Exemple #1
0
int main (int argc, char *argv[]) {

    struct DocumentIndexFormat DocumentIndexPost;
    int PopRankextern;
    int PopRankintern;
    int PopRanknoc;
    int PopRanindex;
    char ShortRank;
    FILE *FH;
    struct popl popextern;
    struct popl popintern;
    struct popl popnoc;
    struct popl popindex;
    uLong htmlBufferSize = 0;
    char *htmlBuffer = NULL;
    char *acl_allowbuffer = NULL;
    char *acl_deniedbuffer = NULL;

    char timebuf[26];

    int optShowhtml = 0;
    int optShowWords = 0;
    int optSummary = 0;
    int optAnchor = 0;
    int optResource = 0;
    int optPopRank = 0;
    int optDelete = 0;
    int optAdult = 0;

    unsigned int DocID;
    char *subname;

    if (getenv("QUERY_STRING") == NULL) {

        extern char *optarg;
        extern int optind, opterr, optopt;
        char c;
        while ((c=getopt(argc,argv,"hwsarpdu"))!=-1) {
            switch (c) {
            case 'h':
                optShowhtml = 1;
                break;
            case 'u':
                optAdult = 1;
                break;
            case 'w':
                optShowWords = 1;
                break;
            case 's':
                optSummary = 1;
                break;
            case 'a':
                optAnchor = 1;
                break;
            case 'p':
                optPopRank = 1;
                break;
            case 'r':
                optResource = 1;
                break;
            case 'd':
                optDelete = 1;
                break;
            default:
                exit(1);
            }
        }
        --optind;

#ifdef DEBUG
        printf("argc %i, optind %i\n",argc,optind);
#endif

        if ((argc - optind)!= 3) {
            printf("Dette programet gir info om en DocID\n\n\tUsage PageInfo DocID collection\n");
            exit(1);
        }


        DocID = atol(argv[1 +optind]);
        subname = argv[2 +optind];


    }
    else {
        printf("Content-type: text/plain\n\n");
        int res;
        // Initialize the CGI lib
        res = cgi_init();

        // Was there an error initializing the CGI???
        if (res != CGIERR_NONE) {
            printf("Error # %d: %s<p>\n", res, cgilib_strerror(res));
            fprintf(stderr,"Cgi-lib error.");
            return -1;
        }

        if (cgi_getentrystr("subname") == NULL) {
            fprintf(stderr,"Didn't recieve any subname.");
            return -1;
        }
        else {
            subname = cgi_getentrystr("subname");
        }

        if (cgi_getentrystr("DocID") == NULL) {
            fprintf(stderr,"Didn't recieve any DocID.");
            return -1;
        }
        else {
            DocID = atol( cgi_getentrystr("DocID") );
        }

    }

    html_parser_init();

    printf("Showing data for Collection \"%s\", DocID %u\n\n",subname,DocID);


    printf("Lot: %i\n",rLotForDOCid(DocID));

    if (optDelete) {
        memset(&DocumentIndexPost,'\0',sizeof(DocumentIndexPost));
        DIWrite(&DocumentIndexPost,DocID,subname,NULL);

        return 0;
    }

    if (DIRead_fmode(&DocumentIndexPost,DocID,subname,'s')) {

        printf("Url: \"%s\"\nLanguage: %s (id: %s)\nOffensive code: %hu\nDocument type: %s\nTime tested sins last good crawl: %hu\nAdult weight: %hu\nResource size: %u\nIP Address: %u\nHtml size: %i\nImage size: %i\nUser ID: %i\nCrawler version: %f\nRepository pointer: %u\n",

               DocumentIndexPost.Url,
               getLangCode2(atoi(DocumentIndexPost.Sprok)),
               DocumentIndexPost.Sprok,
               DocumentIndexPost.Offensive_code,
               DocumentIndexPost.Dokumenttype,
               DocumentIndexPost.AntallFeiledeCrawl,
               DocumentIndexPost.AdultWeight,
               DocumentIndexPost.ResourceSize,
               DocumentIndexPost.IPAddress,
               DocumentIndexPost.htmlSize2,
               DocumentIndexPost.imageSize,
               DocumentIndexPost.userID,
               DocumentIndexPost.clientVersion,
               DocumentIndexPost.RepositoryPointer);

        if (DocumentIndexPost.response == 200) {
            printf("HTTP response: %hu\n",DocumentIndexPost.response);
        }
        else {
            printf("HTTP response: \033[1;31m%hu\033[0m\n",DocumentIndexPost.response);

        }


        ctime_r((time_t *)&DocumentIndexPost.CrawleDato,timebuf);
        timebuf[24] = '\0';


        printf("Last crawled time: %u\n",DocumentIndexPost.CrawleDato);
        printf("Last crawled time ISO: %s\n",timebuf);

        printf("crc32: %u\n",DocumentIndexPost.crc32);

#ifdef BLACK_BOX
        printf("Last seen Unix: %u\n",DocumentIndexPost.lastSeen);
        printf("Last seen ISO: %s", ctime(&DocumentIndexPost.lastSeen));
#endif

        printf("Nr of out links: %u\n",(unsigned int)DocumentIndexPost.nrOfOutLinks);


        char *metadesc, *title, *body;
        if (DocumentIndexPost.SummarySize == 0) {
            printf("Summary: Don't have pre-parsed summery (summary size is 0)\n");

        }
        else if (rReadSummary(DocID,&metadesc, &title, &body,DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize,subname)) {
            printf("\nSummary:\n");
            printf("\tSummary pointer: %u\n\tSummary size: %hu\n",DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize);

            printf("\tTitle from summary:  \"%s\"\n\tMeta description from summary: \"%s\"\n",title,metadesc);
            if (optSummary) {
                printf("Summary body\n*******************\n%s\n*******************\n\n",body);
            }
        }
        else {
            printf("Don't have pre-parsed summery\n");
        }




        struct ReposetoryHeaderFormat ReposetoryHeader;
        char *url, *attributes;

        if (!rReadHtml(&htmlBuffer,&htmlBufferSize,DocumentIndexPost.RepositoryPointer,DocumentIndexPost.htmlSize2,DocID,subname,&ReposetoryHeader,&acl_allowbuffer,&acl_deniedbuffer,DocumentIndexPost.imageSize, &url, &attributes)) {
            printf("rReadHtml: did not returne true!\n");
            return;
        }
        printf("Entire url: %s\n", url);

#ifdef BLACK_BOX
        printf("acl allow raw: \"%s\"\n",acl_allowbuffer);
        printf("acl denied raw: \"%s\"\n",acl_deniedbuffer);

        printf("acl allow resolved: \"%s\"\n",aclResolv(acl_allowbuffer));
        printf("acl denied resolved: \"%s\"\n",aclResolv(acl_deniedbuffer));

        printf("PopRank: %d\n", ReposetoryHeader.PopRank);
#endif

        if (optShowhtml) {

            printf("html uncompresed size %i\n",htmlBufferSize);
            printf("html buff:\n*******************************\n");
            fwrite(htmlBuffer,htmlBufferSize,1,stdout);
            printf("\n*******************************\n\n");


        }
        if (optShowWords) {
            printf("words:\n");
            //run_html_parser( DocumentIndexPost.Url, htmlBuffer, htmlBufferSize, fn );
            char *title, *body;
            html_parser_run(url,htmlBuffer, htmlBufferSize,&title, &body,fn,NULL );
        }
        if (optResource) {
            char buf[500000];
            printf("Resource:\n");
            printf("Ptr: 0x%x Len: %x\n", DocumentIndexPost.ResourcePointer, DocumentIndexPost.ResourceSize);
            if (getResource(rLotForDOCid(DocID), subname, DocID, buf, sizeof(buf)) == 0) {
                printf("\tDid not get any resource\n");
                warn("");
            } else {
                printf("%s\n", buf);
            }
        }

        printf("attributes:\"%s\"\n", attributes);

        free(url);
        free(attributes);
        free(acl_allowbuffer);
        free(acl_deniedbuffer);
    }
    else {
        printf("Cant read post\n");
    }

#ifndef BLACK_BOX

    if (optAdult) {
        int httpResponsCodes[nrOfHttpResponsCodes];
        //char *title;
        //char *body;
        struct adultFormat *adult;
        struct pagewordsFormat *pagewords = malloc(sizeof(struct pagewordsFormat));
        int AdultWeight;
        unsigned char langnr;
        if ((adult = malloc(sizeof(struct adultFormat))) == NULL) {
            perror("malloc argstruct.adult");
            exit(1);
        }

        wordsInit(pagewords);
        langdetectInit();
        adultLoad(adult);

        AdultWeight -1;

        handelPage(pagewords,&ReposetoryHeader,htmlBuffer,htmlBufferSize,&title,&body);

        wordsMakeRevIndex(pagewords,adult,&AdultWeight,&langnr);

        printf("adult %i\n",AdultWeight);
    }

    if (optAnchor) {
        int anchorBufferSize;
        char *anchorBuffer;

        anchorBufferSize = anchorRead(rLotForDOCid(DocID),subname,DocID,NULL,-1);
        anchorBufferSize += 1;
        anchorBuffer = malloc(anchorBufferSize);
        anchorRead(rLotForDOCid(DocID),subname,DocID,anchorBuffer,anchorBufferSize);

        printf("#######################################\nanchors:\n%s\n#######################################\n",anchorBuffer);

        free(anchorBuffer);
    }



    if (optPopRank) {
        popopen (&popindex,"/home/boitho/config/popindex");
        PopRanindex = popRankForDocID(&popindex,DocID);
        popclose(&popindex);
        printf("popindex %i\n",PopRanindex);

        if (popopen (&popextern,"/home/boitho/config/popextern")) {
            PopRankextern =  popRankForDocID(&popextern,DocID);
            printf("PopRankextern: %i\n",PopRankextern);
            popclose(&popextern);
        }
        if (popopen (&popintern,"/home/boitho/config/popintern")) {
            PopRankintern =  popRankForDocID(&popintern,DocID);
            printf("PopRankintern %i\n",PopRankintern);
            popclose(&popintern);
        }
        if (popopen (&popnoc,"/home/boitho/config/popnoc")) {
            PopRanknoc =  popRankForDocID(&popnoc,DocID);
            printf("PopRanknoc %i\n",PopRanknoc);
            popclose(&popnoc);
        }
        if (popopen (&popindex,"/home/boitho/config/popindex")) {
            PopRanindex = popRankForDocID(&popindex,DocID);
            printf("popindex %i\n",PopRanindex);
            popclose(&popindex);
        }



        printf("PopRankextern: %i\nPopRankintern %i\nPopRanknoc %i\n",PopRankextern,PopRankintern,PopRanknoc);


        int brank;
        popopenMemArray_oneLot(subname,rLotForDOCid(DocID));
        brank = popRankForDocIDMemArray(DocID);
        printf("brank %i\n",brank);
        //short rank
        if ( (FH = fopen(SHORTPOPFILE,"rb")) == NULL ) {
            perror("open");
        }
        else {
            if ((fseek(FH,DocID* sizeof(ShortRank),SEEK_SET) == 0) && (fread(&ShortRank,sizeof(ShortRank),1,FH) != 0)) {

                printf("Short rank %u\n",(unsigned char)ShortRank);
            }
            else {
                printf("no hort rank avalibal\n");
            };

            fclose(FH);
        }
    } // if optPopRank
#endif


}
Exemple #2
0
int main (int argc, char *argv[]) {

    int lotNr;
    char lotServer[64];
    int pageCount;
    int i;

    FILE *ADULTWEIGHTFH;
    unsigned char awvalue;

    unsigned int FiltetTime;
    unsigned int FileOffset;
    off_t DocIDPlace;

    char htmlcompressdbuffer[524288];  //0.5 mb
    char imagebuffer[524288];  //0.5 mb

    int httpResponsCodes[nrOfHttpResponsCodes];

    struct ReposetoryHeaderFormat ReposetoryHeader;
    struct DocumentIndexFormat DocumentIndexPost;
    unsigned long int radress;
    FILE *revindexFilesHa[NrOfDataDirectorys];
    struct adultFormat adult;
    unsigned int lastIndexTime;

    if (argc < 2) {
        printf("Dette programet analyserer en url. Usage:\n\tbaddsPageAnalyser url\n");
        exit(0);
    }

    char url[1024];

    char *HtmlBuffer = NULL;


    adultLoad(&adult);

    strncpy(url,argv[1],sizeof(url));

    httpGet(&HtmlBuffer,url);

    //setter opp en alarm slik at run_html_parser blir avbrut hvis den henger seg
    alarm_got_raised = 0;
    signal(SIGALRM, html_parser_timout);

    alarm( 5 );
    //parser htmlen

    run_html_parser(url , HtmlBuffer, strlen(HtmlBuffer), fn );
    alarm( 0);
    if(alarm_got_raised) {
        printf("run_html_parser did time out.\n");
    }
    else {

    }

    unsigned short AdultWeight;
    wordsMakeRevIndex(&adult,&AdultWeight);

    //sortere på forekomst
    pagewordsSortOnOccurrence();

    for(i=0; i<pagewords.revIndexnr; i++) {
        printf("%lu forekomster %i (+1) '%s'\n",pagewords.revIndex[i].WordID,pagewords.revIndex[i].nr,pagewords.revIndex[i].word);

    }

    return 0;
}
Exemple #3
0
void handelPage(char lotServer[], unsigned int LotNr,struct ReposetoryHeaderFormat *ReposetoryHeader, char htmlcompressdbuffer[],char imagebuffebuffer[],FILE *revindexFilesHa[],struct DocumentIndexFormat *DocumentIndexPost, int DocID,int httpResponsCodes[], struct adultFormat *adult) {

        unsigned int HtmlBufferSize;
        char HtmlBuffer[524288];
	int nerror;
	//temp: må finne lang
	char lang[4] = " aa";

		if ((*ReposetoryHeader).response < nrOfHttpResponsCodes) {
			++httpResponsCodes[(*ReposetoryHeader).response];
		}


		//printf("%lu %s\n",(*ReposetoryHeader).DocID, (*ReposetoryHeader).url);

		if (((*ReposetoryHeader).response >= 200) && ((*ReposetoryHeader).response <= 299)) {

		

			HtmlBufferSize = sizeof(HtmlBuffer);
                	if ( (nerror = uncompress((Bytef*)&HtmlBuffer,(uLong *)&HtmlBufferSize,htmlcompressdbuffer,(*ReposetoryHeader).htmlSize)) != 0) {
	                	#ifndef NOWARNINGS
					printf("uncompress error. Code: %i\n",nerror);
				#endif
                	}
                	else {

			
				//if ((*ReposetoryHeader).DocID == 363093661) {
				//	FILE *FH;
				//	FH = fopen("/tmp/IndexerLotdump.html","wb");
				//	fwrite(HtmlBuffer,HtmlBufferSize,1,FH);
				//	fclose(FH);
				//	exit(1);
				//}

		
				
				//begynner på en ny side
				wordsReset();

				if (strcmp((*ReposetoryHeader).content_type,"htm") == 0) {

					//setter opp en alarm slik at run_html_parser blir avbrut hvis den henger seg 
					alarm_got_raised = 0;
					signal(SIGALRM, html_parser_timout);
					
					alarm( 5 );
					//parser htmlen
					run_html_parser( (*ReposetoryHeader).url, HtmlBuffer, HtmlBufferSize, fn );
					alarm( 0);
					if(alarm_got_raised) {
						printf("run_html_parser did time out. At DocID %lu\n",(*ReposetoryHeader).DocID);
					}
					else {

						wordsMakeRevIndex(adult,&(*DocumentIndexPost).AdultWeight);



						revindexFilesAppendWords(revindexFilesHa,(*ReposetoryHeader).DocID,lang);
					}
				
				}
				else if (strcmp((*ReposetoryHeader).content_type,"btx") == 0) {
					//ToDo: definer Boitho TeXt formatet
				}
				else {
					printf("unknown content_type \"%s\"\n",(*ReposetoryHeader).content_type);
				}


			}

		}
		else { //not in 200->299 range
			//radress må vel ikke nulles her, like går å ha den pekene til strukturen i reposetoryet 
			HtmlBufferSize = 0;
		}

			

}