int main (int argc, char *argv[]) { struct config_t maincfg; char c; int noFork = 0; int breakAfter = 0; while ((c=getopt(argc,argv,"m:s"))!=-1) { switch (c) { case 'm': breakAfter = atoi(optarg); break; case 's': noFork = 1; break; default: fprintf(stderr, "Unknown argument: %c", c); errx(1, "Unknown argument: %c", c); } } printf("crawlManager: running maincfgopen\n"); maincfg = maincfgopen(); printf("crawlManager: running maincfg_get_int\n"); int bbdnport = maincfg_get_int(&maincfg,"BLDPORT"); sconnect(connectHandler, bbdnport, noFork, breakAfter); printf("bbdnserver:Main() sconnect ferdig\n"); maincfgclose(&maincfg); return 0; }
int main(int argc, char **argv, char **envp) { int postsize; char *xmldata, *data; xmlDocPtr doc; xmlNodePtr cur, anode; int bbdnsock; int bbdnport; char *status = NULL; struct config_t maincfg; /* Read in config file */ maincfg = maincfgopen(); bbdnport = maincfg_get_int(&maincfg, "BLDPORT"); maincfgclose(&maincfg); key_get(systemkey); char *request_method; if (!bbdn_conect(&bbdnsock, "", bbdnport)) cgi_error(500, bstrerror()); request_method = getenv("HTTP_REQUEST_METHOD")!=NULL ? strdup(getenv("HTTP_REQUEST_METHOD")) : strdup(getenv("REQUEST_METHOD")); // We will handel the post stuf our self. Set REQUEST_METHOD to GET so cgi-util ignores it. setenv("REQUEST_METHOD", "GET", 1); if (cgi_init() != CGIERR_NONE) { cgi_error(500, "Can't init cgi-util"); } /* * Either called from command line, and then we want a file. * or a http get/put * Or we are handling a web request, and getting the data from stdin. */ if ((cgi_getentrystr("method") != NULL) && (strcmp(cgi_getentrystr("method"),"rest") == 0)) { char api[100], coll[100], url[512]; char *requrle; if (getenv("REQUEST_URI") == NULL) { cgi_error(500, "Can't read REQUEST_URI"); } requrle = strdup(getenv("REQUEST_URI")); unescape_url(requrle); sscanf(requrle,"/%[a-z]/%[a-zA-Z0-9_-]/%[^?]", api, coll, url); #ifdef DEBUG fprintf(stderr, "api: \"%s\"\n",api); fprintf(stderr, "coll: \"%s\"\n",coll); fprintf(stderr, "url: \"%s\"\n",url); fprintf(stderr, "request_method: \"%s\"\n",request_method); fprintf(stderr, "reques url \"%s\"\n",getenv("REQUEST_URI")); fprintf(stderr, "reques url unescaped \"%s\"\n",requrle); #endif free(requrle); if (strcmp(request_method,"POST") == 0 || strcmp(request_method,"ADDDELAYED") == 0 || strcmp(request_method,"PUT") == 0) { if (getenv("CONTENT_LENGTH") == NULL) { cgi_error(500, "Can't read CONTENT_LENGTH"); } // Get data length postsize = atoi(getenv("CONTENT_LENGTH")); data = malloc(postsize + 1); if (data == NULL) { cgi_error(500, "Can't allocate data."); } // Read data fread(data, 1, postsize, stdin); data[postsize] = '\0'; // add in to repo if (bbdn_docadd(bbdnsock, coll, // collection name url, // url cgi_getentrystr("documenttype"), // document type data, // data postsize, // data size 0, // lastmodified cgi_getentrystr("acl_allow")!=NULL ? cgi_getentrystr("acl_allow") : "Everyone", // acl allow cgi_getentrystr("acl_denied"), // acl denied cgi_getentrystr("title"), // title cgi_getentrystr("documentformat"), // document format cgi_getentrystr("attributes"), // attributes NULL, // image 0 // image size ) != 1) { cgi_error(500, "bbdn_docadd() failed. Can't add document."); } if (strcmp(request_method,"ADDDELAYED") != 0) { // close it sd_close(bbdnsock, coll); } asprintf(&status,"Added %s to %s\n",url,coll); } else if (strcmp(request_method,"DELETE") == 0) { if (url[0] == '\0') { if (sd_deletecollection(bbdnsock, coll) != 1) { cgi_error(500, "Can't delete collection"); } asprintf(&status,"Deleted collection %s\n",coll); } else { if (bbdn_deleteuri(bbdnsock, coll, url) != 1) { cgi_error(500, "Can't delete document"); } asprintf(&status,"Deleted url %s in %s\n",url,coll); } } else if (strcmp(request_method,"CLOSE") == 0) { sd_close(bbdnsock, coll); asprintf(&status,"Closed %s\n",coll); } else { cgi_error(500, "Unknown request method \"%s\"", request_method ); } #ifdef DEBUG // Print the envirement so we can better see what is going on. char** env; for (env = envp; *env != 0; env++) { char* thisEnv = *env; fprintf(stderr, "%s\n", thisEnv); } #endif } else if ((cgi_getentrystr("do") != NULL) && (strcmp(cgi_getentrystr("do"),"add") == 0)) { char *data; int datasize; int n; const char *url = getenv("HTTP_X_FILENAME") ? getenv("HTTP_X_FILENAME") : cgi_getentrystr("url"); const char *coll = cgi_getentrystr("collection"); if (url == NULL) { cgi_error(500, "No url specified. Either set http header HTTP_X_FILENAME or get parameter 'url'.\n"); } if (coll == NULL) { cgi_error(500, "No collection specified\n"); } char *tmpname; FILE *fh; asprintf(&tmpname,"/tmp/%s",url); fh = fopen(tmpname,"wb"); if (fh == NULL) { cgi_error(500, "Can't open file %s",tmpname); } if ((data = malloc( atoi(getenv("CONTENT_LENGTH")) )) == NULL) { cgi_error(500, "Can't malloc data"); } datasize = 0; while ((n = fread ((unsigned char *)(data + datasize),1,1024,stdin)) > 0) { datasize += n; } fwrite(data,1,datasize,fh); fclose(fh); free(tmpname); // bbdn_docadd(bbdnsock, xmldoc.collection, uri, xmldoc.documenttype, xmldoc.body, xmldoc.bodysize, // xmldoc.lastmodified, xmldoc.aclallow, xmldoc.acldeny, xmldoc.title, xmldoc.documentformat, xmldoc.attributes, image, image_size); bbdn_docadd(bbdnsock, coll, url, "", data, datasize, 0, "Everyone", "", "omp1", "", "", NULL, 0); // close it sd_close(bbdnsock, coll); } else if ((cgi_getentrystr("do") != NULL) && (strcmp(cgi_getentrystr("do"),"delete") == 0)) { const char *url = getenv("HTTP_X_FILENAME") ? getenv("HTTP_X_FILENAME") : cgi_getentrystr("url"); const char *coll = cgi_getentrystr("collection"); if (url == NULL) { cgi_error(500, "No url specified. Either set http header HTTP_X_FILENAME or get parameter 'url'.\n"); } if (coll == NULL) { cgi_error(500, "No collection specified\n"); } bbdn_deleteuri(bbdnsock, coll, url); asprintf(&status,"%s deleted.\n", url); } else if (getenv("CONTENT_LENGTH") != NULL) { // Get data length postsize = atoi(getenv("CONTENT_LENGTH")); xmldata = malloc(postsize + 1); // Read data fread(xmldata, 1, postsize, stdin); xmldata[postsize] = '\0'; //fprintf(stderr, "Received %i bytes.\n", postsize); //fprintf(stderr, "Got document:\n%s\n", xmldata); //parsing xml doc = xmlParseDoc((xmlChar*)xmldata); if (doc == NULL) cgi_error(500, "Unable to parse document"); cur = xmlDocGetRootElement(doc); if (cur == NULL) { xmlFreeDoc(doc); cgi_error(500, "empty document"); } // Some document checking if (xmlStrcmp(cur->name, (const xmlChar *)ROOT_NODE_NAME)) { xmlFreeDoc(doc); cgi_error(500, "document of the wrong type, root node != %s, but %s\n", ROOT_NODE_NAME, cur->name); } if ((anode = xml_find_child(cur, "key")) != NULL) { char *p; p = (char *)xmlNodeListGetString(doc, anode->xmlChildrenNode, 1); if (p == NULL) cgi_error(500, "No key data"); if ((systemkey[0] != '\0') && (!key_equal(systemkey, p))) { cgi_error(500, "Keys does not match: Got \"%s\" but wanted \"%s\"\n",p,systemkey); } } else { cgi_error(500, "Did not receive a key"); } if ((anode = xml_find_child(cur, "version")) != NULL) { xmlChar *p; p = xmlNodeListGetString(doc, anode->xmlChildrenNode, 1); version = atoi((char*)p); xmlFree(p); } else { cgi_error(500, "Did not receive a version number"); } for (cur = cur->xmlChildrenNode; cur != NULL; cur = cur->next) { if ((!xmlStrcmp(cur->name, (const xmlChar *) "key"))){ // Ignore } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "version"))){ // Ignore } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "add"))){ xml_add(bbdnsock, doc, cur); } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "delete"))){ xml_delete(bbdnsock, doc, cur); } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "close"))) { xml_close(bbdnsock, doc, cur); } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "create"))) { xml_create(bbdnsock, doc, cur); } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "users"))) { xml_users(doc, cur); } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "gcwhispers"))) { xml_gcwhispers(bbdnsock, doc, cur); } else if ((!xmlStrcmp(cur->name, (const xmlChar *) "error"))) { xml_errormsg(bbdnsock, doc, cur); } else if ((!xmlStrcmp(cur->name, (xmlChar*)"text"))) { //fprintf(stderr, "Got text: %s\n", xmlNodeListGetString(doc, cur, 1)); // Ignore for now } else { warnx("Unknown xml node '%s'", cur->name); } } } else { cgi_error(500, "Didn't receive any command or data."); } if (status != NULL) { printf("Content-type: text/plain\n\n"); printf(status); } else { cgi_error(500, "Reached end of program without status."); } return 0; }
int main(int argc, char *argv[]) { bblog_init("searchd"); bblog(CLEAN, "Initializing..."); int sockfd; int runCount; //int newsockfd; socklen_t clilen; struct sockaddr_in cli_addr, serv_addr; FILE *LOGFILE; //FILE *LOCK; struct searchd_configFORMAT searchd_config; struct config_t maincfg; searchd_config.searchport = 0; searchd_config.optLog = 0; searchd_config.optMax = 0; searchd_config.optSingle = 0; searchd_config.optrankfile = NULL; searchd_config.optPreOpen = 0; searchd_config.optFastStartup = 0; searchd_config.optCacheIndexes = 1; searchd_config.optAlarm = 60; // Needed for the speller to properly convert utf8 to wchar_t setlocale(LC_ALL, "en_US.UTF-8"); /* Ignore collection updates for now */ signal(SIGUSR2, SIG_IGN); /* And ignore spelling updates */ signal(SIGUSR1, SIG_IGN); char c; while ((c=getopt(argc,argv,"clp:m:b:vsofA:L:S:a:"))!=-1) { switch (c) { case 'p': searchd_config.searchport = atoi(optarg); bblog(CLEAN, "searchd: Option -p: Using port %i.",searchd_config.searchport); break; case 'l': searchd_config.optLog = 1; break; case 'o': searchd_config.optPreOpen = 1; break; case 'm': searchd_config.optMax = atoi(optarg); break; case 'b': searchd_config.optrankfile = optarg; break; case 'v': /* XXX: Remove now that we have severity in the logger? */ bblog(CLEAN, "searchd: Option -v: Verbose output."); globalOptVerbose = 1; break; case 's': bblog(INFO, "Option -s: Won't fork for new connections"); searchd_config.optSingle = 1; break; case 'f': searchd_config.optFastStartup = 1; break; case 'c': searchd_config.optCacheIndexes = 0; break; case 'A': bblog_set_appenders(atoi(optarg)); break; case 'L': bblog_set_severity(atoi(optarg)); break; case 'S': spelling_min_freq = strtol(optarg, NULL, 10); break; case 'a': searchd_config.optAlarm = atoi(optarg); break; default: bblog(ERROR, "Unknown argument: %c", c); errx(1, "Unknown argument: %c", c); } } #ifdef BLACK_BOKS bblog(CLEAN, "Blackbox mode (searchdbb)"); time_t starttime; time(&starttime); if (searchd_config.optLog) { /* Only add file logging if syslog is disabled */ if ((bblog_get_appenders() & LOGGER_APPENDER_SYSLOG) == 0) bblog_set_appenders(LOGGER_APPENDER_FILE|bblog_get_appenders()); } /* Write pidfile */ FILE *pidfile = fopen(bfile("var/searchd.pid"), "w"); if (pidfile != NULL) { fprintf(pidfile, "%d", getpid()); fclose(pidfile); } else { bblog(WARN, "Unable to write to pidfile"); } bblog(CLEAN, "searchd: Starting. Time is %s",ctime(&starttime)); #endif #ifdef DEBUG bblog(DEBUGINFO, "searchd: Debug: argc %i, optind %i",argc,optind); #endif if (searchd_config.optrankfile == NULL) { searchd_config.optrankfile = "Brank"; } #ifdef WITH_SPELLING if (searchd_config.optFastStartup != 1) { if ((spelling = train(bfile("var/dictionarywords"))) == NULL) { bblog(ERROR, "Can't init spelling."); } cache_spelling_keepalive(&spelling); signal(SIGUSR1, cache_spelling_hup); } #endif if (argc > optind) { strncpy(servername,argv[optind], sizeof(servername) -1); } else { bblog(ERROR, "No hostname supplied"); errx(1, "You have to supply a hostname"); } lotPreOpenStartl(&searchd_config.lotPreOpen.DocumentIndex,"DocumentIndex","www",searchd_config.optPreOpen); lotPreOpenStartl(&searchd_config.lotPreOpen.Summary,"summary","www",searchd_config.optPreOpen); #ifdef BLACK_BOKS if (searchd_config.optCacheIndexes == 1) { if (searchd_config.optFastStartup != 1) { bblog(INFO, "Reading indexes"); cache_indexes(0); bblog(INFO, "Cached indexes: %dMB, cached indexes: %d", indexcachescached[0]/(1024*1024), indexcachescached[1]); preopen(); cache_fresh_lot_collection(); cache_indexes_keepalive(); signal(SIGUSR2, cache_indexes_hup); } else { signal(SIGUSR2, SIG_IGN); } } else { signal(SIGUSR2, SIG_IGN); } #endif maincfg = maincfgopen(); if (searchd_config.searchport == 0) { searchd_config.searchport = maincfg_get_int(&maincfg,"BSDPORT"); } searchd_config.cmc_port = maincfg_get_int(&maincfg,"CMDPORT"); maincfgclose(&maincfg); /***********************************************************************************/ //prøver å få fil lock. Bare en deamon kan kjøre avgangen /* #ifndef ALLOW_MULTIPLE_SEARCHD if ((LOCK = fopen("/tmp/searchd.loc","w")) == NULL) { perror("lock file"); exit(1); } if (flock(fileno(LOCK),LOCK_EX | LOCK_NB) != 0) { if (errno == EWOULDBLOCK) { printf("En annen prosses kjører allerede. Steng denne først.\n"); } else { perror("cant get lock file"); } exit(1); } #endif */ /***********************************************************************************/ //#ifndef BLACK_BOKS /* Initialize the configuration */ config_init(&cfg); /* Load the file */ #ifdef DEBUG bblog(DEBUGINFO, "searchd: Debug: Loading [%s] ...",bfile(cfg_searchd)); #endif if (!config_read_file(&cfg, bfile(cfg_searchd))) { bblog(ERROR, "config read failed: [%s]: %s at line %i",bfile(cfg_searchd),config_error_text(&cfg),config_error_line(&cfg)); exit(1); } //#endif html_parser_init(); /* #ifdef WITH_THREAD pthread_t chld_thr; printf("starting whth thread\n"); #else printf("starting single thread version\n"); #endif */ bblog(CLEAN, "Servername: %s", servername); //ToDo: må ha låsing her if ((LOGFILE = bfopen("config/query.log","a")) == NULL) { bblog_errno(ERROR, "%s", bfile("config/query.log")); } else { fprintf(LOGFILE,"starting server %s\n",servername); fclose(LOGFILE); } #ifdef BLACK_BOKS // Initialiser thesaurus med ouput-filene fra 'build_thesaurus_*': searchd_config.thesaurus_all = NULL; #ifndef WITHOUT_THESAURUS bblog(INFO, "init thesaurus"); searchd_config.thesaurus_all = NULL; if (searchd_config.optFastStartup != 1) { searchd_config.thesaurus_all = load_all_thesauruses(bfile("data/thesaurus/")); if (searchd_config.thesaurus_all == NULL) { bblog(ERROR, "Unable to open thesaurus. Disabling stemming"); } else { bblog(INFO, "init thesaurus done"); } } #endif bblog(INFO, "init file-extensions"); searchd_config.getfiletypep = fte_init(bfile("config/file_extensions.conf")); if (searchd_config.getfiletypep == NULL) { bblog(ERROR, "Unable to open file-extensions configuration file. Disabling file-extensions."); } bblog(INFO, "init attribute descriptions"); searchd_config.attrdescrp = adf_init(bfile("config/attribute_descriptions.conf")); if (searchd_config.attrdescrp == NULL) { bblog(ERROR, "Unable to open attribute descriptions configuration file. Disabling attribute descriptions."); } bblog(INFO, "init show-attributes"); char *warnings; /*searchd_config.showattrp = show_attributes_init(bfile("config/show_attributes.conf"), &warnings); if (searchd_config.showattrp == NULL) { fprintf(stderr, "searchd: ERROR!! Unable to open show-attributes configuration file. Disabling attributes.\n"); } else if (warnings[0]!='\0') { fprintf(stderr, "searchd: ******************* Warnings reading show-attributes config: ********************\n"); fprintf(stderr, "%s", warnings); fprintf(stderr, "searchd: *********************************************************************************\n"); }*/ #else //starter opp bblog(INFO, "Loading domain-ids..."); iintegerLoadMemArray2(&global_DomainIDs,"domainid",sizeof(unsigned short), "www"); //laster inn alle poprankene bblog(INFO, "Loading pop MemArray..."); popopenMemArray2("www",searchd_config.optrankfile); // ToDo: hardkoder subname her, da vi ikke vet siden vi ikke her får et inn enda bblog(INFO, "Loading adultWeight MemArray..."); adultWeightopenMemArray2("www"); // ToDo: hardkoder subname her, da vi ikke vet siden vi ikke her får et inn enda #endif IIndexInaliser(); #ifdef WITH_MEMINDEX IIndexLoad(); #endif if((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { bblog(ERROR, "Server error! Can't open stream socket."); exit(1); } memset((char *) &serv_addr, 0, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; serv_addr.sin_addr.s_addr = htonl(INADDR_ANY); serv_addr.sin_port = htons(searchd_config.searchport); bblog(INFO, "Will bind to port %i",searchd_config.searchport); //seter at sokket kan rebrukes int yes=1; if (setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,&yes,sizeof(int)) == -1) { bblog_errno(ERROR, "setsockopt()"); exit(1); } if(bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { bblog_errno(ERROR, "Can't bind local address. Port %i", searchd_config.searchport); exit(1); } /* set the level of thread concurrency we desire */ //thr_setconcurrency(5); listen(sockfd, 5); runCount = 0; #ifndef NEW_CHILD_CATCHER signal(SIGCLD, SIG_IGN); /* now I don't have to wait() for forked children! */ #else { // runarb 28 juni 2009: // Hvis vi har verbose output så skal vi wait()e for våre barn, og vise prity print når de dør. // desverre har det vært mye kød, der hovedprosessen blir hengene i sigchild_handler() og ikke kan // fork'e flere barn. For å ungå dtte venter vi ikke på våre barn til vanlig. if (globalOptVerbose) { struct sigaction sa; int ret; sa.sa_sigaction = sigchild_handler; sigemptyset(&sa.sa_mask); sa.sa_flags = SA_SIGINFO; ret = sigaction(SIGCHLD, &sa, 0); if (ret) { bblog_errno(ERROR, "sigaction()"); exit(1); } } else { signal(SIGCLD, SIG_IGN); /* now I don't have to wait() for forked children! */ } } #endif bblog(CLEAN, "|------------------------------------------------------------------------------------------------|"); bblog(CLEAN, "|%-40s | %-11s | %-11s | %-11s | %-11s|","query", "TotaltTreff", "showabal", "filtered", "total_usecs"); bblog(CLEAN, "|------------------------------------------------------------------------------------------------|"); for((clilen = sizeof(cli_addr));;) { searchd_config.newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen); if(searchd_config.newsockfd < 0) { /* Just restart */ if (errno == EINTR) continue; bblog(WARN, "searchd: Server warning! Accept error"); } else { if (searchd_config.optSingle) { do_chld((void *) &searchd_config); } else { #ifdef DEBUG bblog(DEBUGINFO, "Debug mode; will not fork to new process."); do_chld((void *) &searchd_config); #else /* #ifdef WITH_THREAD //create a new thread to process the incomming request //thr_create(NULL, 0, do_chld, (void *) searchd_config, THR_DETACHED, &chld_thr); pthread_create(&chld_thr, NULL, do_chld, (void *) &searchd_config); //the server is now free to accept another socket request #else do_chld((void *) &searchd_config); #endif */ bblog(DEBUGINFO, "Forking new prosess."); if (fork() == 0) { // this is the child process close(sockfd); // child doesn't need the listener do_chld((void *) &searchd_config); close(searchd_config.newsockfd); bblog(DEBUGINFO, "Terminating child."); exit(0); } else { close(searchd_config.newsockfd); // perent doesn't need the new socket } #endif } } ++runCount; if ((searchd_config.optMax != 0) && (runCount >= searchd_config.optMax)) { //venter på siste trå. Ikke helt optimalt dette, da vi kan ha flere tråer som kjører i paralell /* #ifdef WITH_THREAD pthread_join(chld_thr, NULL); #endif */ bblog(WARN, "have reached Max runs. Exiting..."); break; } } html_parser_exit(); free(searchd_config.lotPreOpen.Summary); free(searchd_config.lotPreOpen.DocumentIndex); adf_destroy(searchd_config.attrdescrp); fte_destroy(searchd_config.getfiletypep); if (searchd_config.optFastStartup != 1) { thesaurus_destroy(searchd_config.thesaurusp); } //freegjør spelling. Trekt, men kjekt av valgring kan finne ut om noe ikke her blirr frigjort. if (searchd_config.optFastStartup != 1) { untrain(&spelling); } return(0); }