コード例 #1
0
void bbdocument_exist_update_di(char subname[],docid DocID) {

	// Update DI with new existed timestamp
	struct DocumentIndexFormat docindex;

	#if defined(BLACK_BOX) && !defined(_24SEVENOFFICE)
		DIRead(&docindex, DocID, subname);
		docindex.lastSeen = time(NULL);
		DIWrite(&docindex, DocID, subname, NULL);
		debug("bbdocument_exist: satte lastSeen \"%s\"",ctime(&docindex.lastSeen));
	#endif

}
コード例 #2
0
ファイル: main.c プロジェクト: bowlofstew/enterprise-search
int main (int argc, char *argv[]) {

    struct DocumentIndexFormat DocumentIndexPost;
    int PopRankextern;
    int PopRankintern;
    int PopRanknoc;
    int PopRanindex;
    char ShortRank;
    FILE *FH;
    struct popl popextern;
    struct popl popintern;
    struct popl popnoc;
    struct popl popindex;
    uLong htmlBufferSize = 0;
    char *htmlBuffer = NULL;
    char *acl_allowbuffer = NULL;
    char *acl_deniedbuffer = NULL;

    char timebuf[26];

    int optShowhtml = 0;
    int optShowWords = 0;
    int optSummary = 0;
    int optAnchor = 0;
    int optResource = 0;
    int optPopRank = 0;
    int optDelete = 0;
    int optAdult = 0;

    unsigned int DocID;
    char *subname;

    if (getenv("QUERY_STRING") == NULL) {

        extern char *optarg;
        extern int optind, opterr, optopt;
        char c;
        while ((c=getopt(argc,argv,"hwsarpdu"))!=-1) {
            switch (c) {
            case 'h':
                optShowhtml = 1;
                break;
            case 'u':
                optAdult = 1;
                break;
            case 'w':
                optShowWords = 1;
                break;
            case 's':
                optSummary = 1;
                break;
            case 'a':
                optAnchor = 1;
                break;
            case 'p':
                optPopRank = 1;
                break;
            case 'r':
                optResource = 1;
                break;
            case 'd':
                optDelete = 1;
                break;
            default:
                exit(1);
            }
        }
        --optind;

#ifdef DEBUG
        printf("argc %i, optind %i\n",argc,optind);
#endif

        if ((argc - optind)!= 3) {
            printf("Dette programet gir info om en DocID\n\n\tUsage PageInfo DocID collection\n");
            exit(1);
        }


        DocID = atol(argv[1 +optind]);
        subname = argv[2 +optind];


    }
    else {
        printf("Content-type: text/plain\n\n");
        int res;
        // Initialize the CGI lib
        res = cgi_init();

        // Was there an error initializing the CGI???
        if (res != CGIERR_NONE) {
            printf("Error # %d: %s<p>\n", res, cgilib_strerror(res));
            fprintf(stderr,"Cgi-lib error.");
            return -1;
        }

        if (cgi_getentrystr("subname") == NULL) {
            fprintf(stderr,"Didn't recieve any subname.");
            return -1;
        }
        else {
            subname = cgi_getentrystr("subname");
        }

        if (cgi_getentrystr("DocID") == NULL) {
            fprintf(stderr,"Didn't recieve any DocID.");
            return -1;
        }
        else {
            DocID = atol( cgi_getentrystr("DocID") );
        }

    }

    html_parser_init();

    printf("Showing data for Collection \"%s\", DocID %u\n\n",subname,DocID);


    printf("Lot: %i\n",rLotForDOCid(DocID));

    if (optDelete) {
        memset(&DocumentIndexPost,'\0',sizeof(DocumentIndexPost));
        DIWrite(&DocumentIndexPost,DocID,subname,NULL);

        return 0;
    }

    if (DIRead_fmode(&DocumentIndexPost,DocID,subname,'s')) {

        printf("Url: \"%s\"\nLanguage: %s (id: %s)\nOffensive code: %hu\nDocument type: %s\nTime tested sins last good crawl: %hu\nAdult weight: %hu\nResource size: %u\nIP Address: %u\nHtml size: %i\nImage size: %i\nUser ID: %i\nCrawler version: %f\nRepository pointer: %u\n",

               DocumentIndexPost.Url,
               getLangCode2(atoi(DocumentIndexPost.Sprok)),
               DocumentIndexPost.Sprok,
               DocumentIndexPost.Offensive_code,
               DocumentIndexPost.Dokumenttype,
               DocumentIndexPost.AntallFeiledeCrawl,
               DocumentIndexPost.AdultWeight,
               DocumentIndexPost.ResourceSize,
               DocumentIndexPost.IPAddress,
               DocumentIndexPost.htmlSize2,
               DocumentIndexPost.imageSize,
               DocumentIndexPost.userID,
               DocumentIndexPost.clientVersion,
               DocumentIndexPost.RepositoryPointer);

        if (DocumentIndexPost.response == 200) {
            printf("HTTP response: %hu\n",DocumentIndexPost.response);
        }
        else {
            printf("HTTP response: \033[1;31m%hu\033[0m\n",DocumentIndexPost.response);

        }


        ctime_r((time_t *)&DocumentIndexPost.CrawleDato,timebuf);
        timebuf[24] = '\0';


        printf("Last crawled time: %u\n",DocumentIndexPost.CrawleDato);
        printf("Last crawled time ISO: %s\n",timebuf);

        printf("crc32: %u\n",DocumentIndexPost.crc32);

#ifdef BLACK_BOX
        printf("Last seen Unix: %u\n",DocumentIndexPost.lastSeen);
        printf("Last seen ISO: %s", ctime(&DocumentIndexPost.lastSeen));
#endif

        printf("Nr of out links: %u\n",(unsigned int)DocumentIndexPost.nrOfOutLinks);


        char *metadesc, *title, *body;
        if (DocumentIndexPost.SummarySize == 0) {
            printf("Summary: Don't have pre-parsed summery (summary size is 0)\n");

        }
        else if (rReadSummary(DocID,&metadesc, &title, &body,DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize,subname)) {
            printf("\nSummary:\n");
            printf("\tSummary pointer: %u\n\tSummary size: %hu\n",DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize);

            printf("\tTitle from summary:  \"%s\"\n\tMeta description from summary: \"%s\"\n",title,metadesc);
            if (optSummary) {
                printf("Summary body\n*******************\n%s\n*******************\n\n",body);
            }
        }
        else {
            printf("Don't have pre-parsed summery\n");
        }




        struct ReposetoryHeaderFormat ReposetoryHeader;
        char *url, *attributes;

        if (!rReadHtml(&htmlBuffer,&htmlBufferSize,DocumentIndexPost.RepositoryPointer,DocumentIndexPost.htmlSize2,DocID,subname,&ReposetoryHeader,&acl_allowbuffer,&acl_deniedbuffer,DocumentIndexPost.imageSize, &url, &attributes)) {
            printf("rReadHtml: did not returne true!\n");
            return;
        }
        printf("Entire url: %s\n", url);

#ifdef BLACK_BOX
        printf("acl allow raw: \"%s\"\n",acl_allowbuffer);
        printf("acl denied raw: \"%s\"\n",acl_deniedbuffer);

        printf("acl allow resolved: \"%s\"\n",aclResolv(acl_allowbuffer));
        printf("acl denied resolved: \"%s\"\n",aclResolv(acl_deniedbuffer));

        printf("PopRank: %d\n", ReposetoryHeader.PopRank);
#endif

        if (optShowhtml) {

            printf("html uncompresed size %i\n",htmlBufferSize);
            printf("html buff:\n*******************************\n");
            fwrite(htmlBuffer,htmlBufferSize,1,stdout);
            printf("\n*******************************\n\n");


        }
        if (optShowWords) {
            printf("words:\n");
            //run_html_parser( DocumentIndexPost.Url, htmlBuffer, htmlBufferSize, fn );
            char *title, *body;
            html_parser_run(url,htmlBuffer, htmlBufferSize,&title, &body,fn,NULL );
        }
        if (optResource) {
            char buf[500000];
            printf("Resource:\n");
            printf("Ptr: 0x%x Len: %x\n", DocumentIndexPost.ResourcePointer, DocumentIndexPost.ResourceSize);
            if (getResource(rLotForDOCid(DocID), subname, DocID, buf, sizeof(buf)) == 0) {
                printf("\tDid not get any resource\n");
                warn("");
            } else {
                printf("%s\n", buf);
            }
        }

        printf("attributes:\"%s\"\n", attributes);

        free(url);
        free(attributes);
        free(acl_allowbuffer);
        free(acl_deniedbuffer);
    }
    else {
        printf("Cant read post\n");
    }

#ifndef BLACK_BOX

    if (optAdult) {
        int httpResponsCodes[nrOfHttpResponsCodes];
        //char *title;
        //char *body;
        struct adultFormat *adult;
        struct pagewordsFormat *pagewords = malloc(sizeof(struct pagewordsFormat));
        int AdultWeight;
        unsigned char langnr;
        if ((adult = malloc(sizeof(struct adultFormat))) == NULL) {
            perror("malloc argstruct.adult");
            exit(1);
        }

        wordsInit(pagewords);
        langdetectInit();
        adultLoad(adult);

        AdultWeight -1;

        handelPage(pagewords,&ReposetoryHeader,htmlBuffer,htmlBufferSize,&title,&body);

        wordsMakeRevIndex(pagewords,adult,&AdultWeight,&langnr);

        printf("adult %i\n",AdultWeight);
    }

    if (optAnchor) {
        int anchorBufferSize;
        char *anchorBuffer;

        anchorBufferSize = anchorRead(rLotForDOCid(DocID),subname,DocID,NULL,-1);
        anchorBufferSize += 1;
        anchorBuffer = malloc(anchorBufferSize);
        anchorRead(rLotForDOCid(DocID),subname,DocID,anchorBuffer,anchorBufferSize);

        printf("#######################################\nanchors:\n%s\n#######################################\n",anchorBuffer);

        free(anchorBuffer);
    }



    if (optPopRank) {
        popopen (&popindex,"/home/boitho/config/popindex");
        PopRanindex = popRankForDocID(&popindex,DocID);
        popclose(&popindex);
        printf("popindex %i\n",PopRanindex);

        if (popopen (&popextern,"/home/boitho/config/popextern")) {
            PopRankextern =  popRankForDocID(&popextern,DocID);
            printf("PopRankextern: %i\n",PopRankextern);
            popclose(&popextern);
        }
        if (popopen (&popintern,"/home/boitho/config/popintern")) {
            PopRankintern =  popRankForDocID(&popintern,DocID);
            printf("PopRankintern %i\n",PopRankintern);
            popclose(&popintern);
        }
        if (popopen (&popnoc,"/home/boitho/config/popnoc")) {
            PopRanknoc =  popRankForDocID(&popnoc,DocID);
            printf("PopRanknoc %i\n",PopRanknoc);
            popclose(&popnoc);
        }
        if (popopen (&popindex,"/home/boitho/config/popindex")) {
            PopRanindex = popRankForDocID(&popindex,DocID);
            printf("popindex %i\n",PopRanindex);
            popclose(&popindex);
        }



        printf("PopRankextern: %i\nPopRankintern %i\nPopRanknoc %i\n",PopRankextern,PopRankintern,PopRanknoc);


        int brank;
        popopenMemArray_oneLot(subname,rLotForDOCid(DocID));
        brank = popRankForDocIDMemArray(DocID);
        printf("brank %i\n",brank);
        //short rank
        if ( (FH = fopen(SHORTPOPFILE,"rb")) == NULL ) {
            perror("open");
        }
        else {
            if ((fseek(FH,DocID* sizeof(ShortRank),SEEK_SET) == 0) && (fread(&ShortRank,sizeof(ShortRank),1,FH) != 0)) {

                printf("Short rank %u\n",(unsigned char)ShortRank);
            }
            else {
                printf("no hort rank avalibal\n");
            };

            fclose(FH);
        }
    } // if optPopRank
#endif


}
コード例 #3
0
ファイル: main.c プロジェクト: dateline/enterprise-search
int main (int argc, char *argv[]) {

        int lotNr;
	char lotServer[64];
	int pageCount;
	int i;

        unsigned int FiltetTime;
        unsigned int FileOffset;

        char htmlcompressdbuffer[524288];  //0.5 mb
        char imagebuffer[524288];  //0.5 mb
	
	int httpResponsCodes[nrOfHttpResponsCodes];

	struct ReposetoryHeaderFormat ReposetoryHeader;
	struct DocumentIndexFormat DocumentIndexPost;
	unsigned long int radress;
	FILE *revindexFilesHa[NrOfDataDirectorys];
	struct adultFormat adult;
	unsigned int lastIndexTime;

        if (argc < 3) {
                printf("Dette programet indekserer en lot. Usage:\n\tIndexerLot lotNr subname\n");
                exit(0);
        }

	for(i=0;i<nrOfHttpResponsCodes;i++) {
		httpResponsCodes[i] = 0;
	}

	lotNr = atoi(argv[1]);
	strncpy(subname,argv[2],sizeof(subname) -1);


	//find server based on lotnr
	lotlistLoad();
	lotlistGetServer(lotServer,lotNr);


	printf("vil index lot nr %i at %s\n",lotNr,lotServer);

	adultLoad(&adult);

	langdetectInit();



	//temp: må hente dette fra slot server eller fil
	FiltetTime = 0;
	FileOffset = 0;

	pageCount = 0;

	if (0) {


		printf("will ges pages by net\n");

		revindexFilesOpenNET(revindexFilesHa);

		while (rGetNextNET(lotServer,lotNr,&ReposetoryHeader,htmlcompressdbuffer,imagebuffer,&radress,FiltetTime,FileOffset,subname)) {

                                global_curentDocID = ReposetoryHeader.DocID;
                                if (strchr(ReposetoryHeader.url,'?') == 0) {
                                        global_curentUrlIsDynamic = 0;
                                }
                                else {
                                        global_curentUrlIsDynamic = 1;
                                }
		
			
			
			handelPage(lotServer,lotNr,&ReposetoryHeader,htmlcompressdbuffer,imagebuffer,revindexFilesHa,&DocumentIndexPost,ReposetoryHeader.DocID,httpResponsCodes,&adult);
			//datta skal uansett kopieres over
			//kopierer over di data
			copyRepToDi(&DocumentIndexPost,&ReposetoryHeader);

			DocumentIndexPost.RepositoryPointer = radress;


			//skiver til DocumentIndex
			DIWriteNET(lotServer,&DocumentIndexPost,ReposetoryHeader.DocID,subname);


			++pageCount;
		
			//temp: 
			//if(pageCount > 999) {
			//	printf("Exeting after only %i docs\n",pageCount);
			//	break;
			//}

		}

		printf("Sending pages\n");

		revindexFilesSendNET(revindexFilesHa,lotNr);

	}
	else {
		printf("Wil acess files localy\n");

		//sjekker om vi har nokk palss
		if (!lotHasSufficientSpace(lotNr,4096,subname)) {
			printf("insufficient disk space\n");
			exit(1);
		}


		//finner siste indekseringstid
		lastIndexTime =  GetLastIndexTimeForLot(lotNr,subname);

		
		if(lastIndexTime != 0) {
			printf("lastIndexTime is not 0, but %i\n",lastIndexTime);
			exit(1);
		}
		
		
		revindexFilesOpenLocal(revindexFilesHa,lotNr,"Main","wb",subname);

		//temp:Søker til problemområdet
		//FileOffset = 334603785;		

		while (rGetNext(lotNr,&ReposetoryHeader,htmlcompressdbuffer,sizeof(htmlcompressdbuffer),imagebuffer,&radress,FiltetTime,FileOffset,subname)) {		
			//printf("D: %u, R: %lu\n",ReposetoryHeader.DocID, radress);


                                global_curentDocID = ReposetoryHeader.DocID;
                                if (strchr(ReposetoryHeader.url,'?') == 0) {
                                        global_curentUrlIsDynamic = 0;
                                }
                                else {
                                        global_curentUrlIsDynamic = 1;
                                }

								
				printf("%s\n",ReposetoryHeader.url);
				
				
				handelPage(lotServer,lotNr,&ReposetoryHeader,htmlcompressdbuffer,imagebuffer,revindexFilesHa,&DocumentIndexPost,ReposetoryHeader.DocID,httpResponsCodes,&adult);

				

				//datta skal kopieres over uanset hva som skjer
				//kopierer over di data
				copyRepToDi(&DocumentIndexPost,&ReposetoryHeader);

				DocumentIndexPost.RepositoryPointer = radress;

				//skiver til DocumentIndex
				DIWrite(&DocumentIndexPost,ReposetoryHeader.DocID,subname);
				

			++pageCount;
		
			//if(pageCount > 9999) {
			//	printf("Exeting after only %i docs\n",pageCount);
			//	//break;
			//	exit(1);
			//}

		}

		//skriver riktig indexstide til lotten
		setLastIndexTimeForLot(lotNr,httpResponsCodes,subname);

		// vi må ikke kopiere revindex filene da vi jobber på de lokale direkte
	}


	langdetectDestroy();

	printf("indexed %i pages\n\n\n",pageCount);

	return 0;
}
コード例 #4
0
ファイル: main.c プロジェクト: bowlofstew/enterprise-search
void connectHandler(int socket) {
    struct packedHedderFormat packedHedder;

    int i,n;
    int LotNr;
    char lotPath[512];
    char buf[100];
    unsigned int FilterTime;
    int filnamelen;
    FILE *FH;
    struct stat inode;      // lager en struktur for fstat å returnere.
    off_t filesize;
    char c;

    struct DocumentIndexFormat DocumentIndexPost;
    int DocID;

    struct ReposetoryHeaderFormat ReposetoryHeader;
    unsigned int radress;

    char htmlbuffer[524288];
    int destLeng;
    char dest[512];

    off_t fileBloks,filerest;
    char *filblocbuff;


    //while ((i=read(socket, &packedHedder, sizeof(struct packedHedderFormat))) > 0) {
    while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) {

        //printf("command: %i\n",packedHedder.command);
        //printf("i er %i\n",i);
        printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
        //printf("subname: %s\n",packedHedder.subname);
        //lar size reflektere hva som er igjen av pakken
        packedHedder.size = packedHedder.size - sizeof(packedHedder);

        if (packedHedder.command == C_rmkdir) {

            printf("C_rmkdir\n");

            //leser data. Det skal væren en int som sier hvilken lot vi vil ha
            if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) {
                perror("Cant read lotnr");
                exit(1);
            }

            //leser destinasjonelengden
            if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) {
                perror("Cant read destLeng");
                exit(1);
            }

            if (destLeng > sizeof(dest)) {
                printf("dest filname is to long at %i\n",destLeng);
                exit(1);
            }

            //leser destinasjonene
            if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) {
                perror("Cant read dest");
                exit(1);
            }

            GetFilPathForLot(lotPath,LotNr,packedHedder.subname);

            sprintf(lotPath,"%s%s",lotPath,dest);

            printf("mkdir %s\n",lotPath);

            makePath(lotPath);

            printf("~C_rmkdir\n");


        }
        else if (packedHedder.command == C_rComand) {


            //leser data. Det skal væren en int som sier hvilken lot vi vil ha
            if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) {
                perror("Cant read lotnr");
                exit(1);
            }

            //leser destinasjonelengden
            if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) {
                perror("Cant read destLeng");
                exit(1);
            }

            if (destLeng > sizeof(dest)) {
                printf("dest filname is to long at %i\n",destLeng);
                exit(1);
            }

            //leser destinasjonene
            if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) {
                perror("Cant read dest");
                exit(1);
            }

            printf("run command %s\n",dest);

            system(dest);

        }
        else if (packedHedder.command == C_getLotToIndex) {
            printf("fikk C_getLotToIndex\n");

            int dirty;

            if ((i=recv(socket, &dirty, sizeof(dirty),MSG_WAITALL)) == -1) {
                perror("Cant read dirty");
                exit(1);
            }

            printf("dirty: %i\n",dirty);

            LotNr = findLotToIndex(packedHedder.subname,dirty);

            printf("sending respons\n");
            sendall(socket,&LotNr, sizeof(LotNr));

        }
        else if (packedHedder.command == C_getlotHasSufficientSpace) {
            printf("fikk C_getLotToIndex\n");

            int needSpace;
            int response;

            if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
                perror("Cant read lotnr");
                exit(1);
            }

            if ((i=recv(socket, &needSpace, sizeof(needSpace),MSG_WAITALL)) == -1) {
                perror("Cant read dirty");
                exit(1);
            }


            printf("needSpace: %i, LotNr %i\n",needSpace,LotNr);


            response = lotHasSufficientSpace(LotNr, needSpace, packedHedder.subname);


            printf("sending respons\n");
            sendall(socket,&response, sizeof(response));

        }
        else if (packedHedder.command == C_rGetSize) {
            printf("fikk C_rGetSize\n");


            //leser data. Det skal væren en int som sier hvilken lot vi vil ha
            if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
                perror("Cant read lotnr");
                exit(1);
            }

            if ((i=read(socket, &filnamelen, sizeof(filnamelen))) == -1) {
                perror("Cant read filnamelen");
                exit(1);
            }

            if (filnamelen > sizeof(buf)) {
                printf("filname to long\n");
            };

            if ((i=read(socket, buf, filnamelen)) == -1) {
                perror("Cant read filnamelen");
                exit(1);
            }

            printf("filname %s\n",buf);

            if ((FH = lotOpenFileNoCasheByLotNr(LotNr,buf,"rb",'s',packedHedder.subname)) == NULL) {
                perror(buf);
                //sending that he fil is emty
                fileBloks = 0;

                sendall(socket,&fileBloks, sizeof(fileBloks));

            }
            else {
                //finner og sender il størelse
                fstat(fileno(FH),&inode);
                //filesize = inode.st_size;
                //sendall(socket,&filesize, sizeof(filesize));

                fileBloks = inode.st_size;

                printf("size is %" PRId64 "\n",fileBloks);

                sendall(socket,&fileBloks, sizeof(fileBloks));

                fclose(FH);
            }
        }
        else if (packedHedder.command == C_rGetFile) {
            printf("fikk C_rGetFile\n");


            //leser data. Det skal væren en int som sier hvilken lot vi vil ha
            if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
                perror("Cant read lotnr");
                exit(1);
            }

            if ((i=read(socket, &filnamelen, sizeof(filnamelen))) == -1) {
                perror("Cant read filnamelen");
                exit(1);
            }

            if (filnamelen > sizeof(buf)) {
                printf("filname to long\n");
            };

            if ((i=read(socket, buf, filnamelen)) == -1) {
                perror("Cant read filnamelen");
                exit(1);
            }

            printf("filname %s\n",buf);

            if ((FH = lotOpenFileNoCasheByLotNr(LotNr,buf,"rb",'s',packedHedder.subname)) == NULL) {
                perror(buf);
                //sending that the fil is emty
                fileBloks = 0;
                filerest = 0;

                sendall(socket,&fileBloks, sizeof(fileBloks));
                sendall(socket,&filerest, sizeof(filerest));

            }
            else {
                //finner og sender fil størelse
                fstat(fileno(FH),&inode);
                //filesize = inode.st_size;
                //sendall(socket,&filesize, sizeof(filesize));

                fileBloks = (int)floor(inode.st_size / rNetTrabsferBlok);
                filerest = inode.st_size - (fileBloks * rNetTrabsferBlok);

                sendall(socket,&fileBloks, sizeof(fileBloks));
                sendall(socket,&filerest, sizeof(filerest));

                printf("sending fil. fileBloks %"PRId64", filerest %"PRId64"\n",fileBloks,filerest);


                filblocbuff = (char *)malloc(rNetTrabsferBlok);
                for(i=0; i < fileBloks; i++) {

                    //fread(filblocbuff,sizeof(c),rNetTrabsferBlok,FH);
                    //fread_all(const void *buf, size_t size, FILE *stream)
                    fread_all(filblocbuff,rNetTrabsferBlok,FH, 4096);

                    if ((n=sendall(socket, filblocbuff, rNetTrabsferBlok)) == -1) {
                        perror("Cant recv dest");
                        exit(1);
                    }

                }

                printf("did recv %i fileBloks\n",i);


                fread(filblocbuff,sizeof(c),filerest,FH);

                if ((n=sendall(socket, filblocbuff, filerest)) == -1) {
                    perror("Cant recv filerest");
                    exit(1);
                }

                free(filblocbuff);


                /*
                   for (i=0;i<filesize;i++) {
                   fread(&c,sizeof(char),1,FH);
                   send(socket, &c, sizeof(char), 0);
                //printf("%i\n",(int)c);
                }
                 */
                printf("send file end\n");

                fclose(FH);
            }

        }
        else if (packedHedder.command == C_rGetNext) {
            printf("fikk C_rGetNext\n");

            printf("støttes ikke lengere");
            exit(1);
            /*
            		//leser data. Det skal væren en unigned int som sier hvilken lot vi vil ha
            		//har deklarert den som int her ???
            		if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
            			perror("Cant read lotnr");
            			exit(1);
            		}
            		printf("leser FilterTime\n");
            		//leser filtertime
            		if ((i=read(socket, &FilterTime, sizeof(FilterTime))) == -1) {
            			perror("Cant read lotnr");
            			exit(1);
            		}

            		printf("lotnr %i FilterTime %u\n",LotNr,FilterTime);

            		//henter inn data om den lotten
            		if (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,NULL,&radress,FilterTime,0)) {

            			//printf("DocId: %i url: %s\n",ReposetoryHeader.DocID,ReposetoryHeader.url);

            			//sender pakke hedder
            			sendpacked(socket,C_rLotData,PROTOCOLVERSION, ReposetoryHeader.htmlSize + sizeof(ReposetoryHeader) +sizeof(radress), NULL,packedHedder.subname);

            			//sennder ReposetoryHeader'en
            			sendall(socket,&ReposetoryHeader, sizeof(ReposetoryHeader));

            			//sender htmlen
            			sendall(socket,&htmlbuffer, ReposetoryHeader.htmlSize);

            			//sender adressen
            			sendall(socket,&radress,sizeof(radress));
            			//printf("data sent\n");

            			//printf("rGetNext: %i\n",ReposetoryHeader.DocID);

            		}
            		else {
            			sendpacked(socket,C_rEOF,PROTOCOLVERSION, 0, NULL,packedHedder.subname);
            			printf("ferdig\n");
            		}
            */
        }
        else if (packedHedder.command == C_DIWrite) {


            if ((i=recv(socket, &DocumentIndexPost, sizeof(struct DocumentIndexFormat),MSG_WAITALL)) == -1) {
                perror("recv");
                exit(1);
            }

            if ((i=recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
                perror("recv");
                exit(1);
            }

            DIWrite(&DocumentIndexPost,DocID,packedHedder.subname, NULL);

            //printf("DIWrite: %i\n",DocID);

        }
        else if (packedHedder.command == C_DIRead) {

            int DocID;
            struct DocumentIndexFormat DocumentIndexPost;

            printf("got commane C_DIRead. sise %i hsize %i ds %i\n",packedHedder.size, sizeof(packedHedder), sizeof(DocID));

            if ((i=recv(socket, &DocID, sizeof(DocID),0)) == -1) {
                perror("recv");
                exit(1);
            }
            //printf("DocID %i\n",DocID);

            //leser inn datan
            //int DIRead (struct DocumentIndexFormat *DocumentIndexPost, int DocID);
            DIRead(&DocumentIndexPost,DocID,packedHedder.subname);

            sendall(socket,&DocumentIndexPost, sizeof(struct DocumentIndexFormat));
        }
        else if (packedHedder.command == C_rGetIndexTime) {

            int Lotnr;
            unsigned int IndexTime;
            if ((i=recv(socket, &LotNr, sizeof(LotNr),0)) == -1) {
                perror("recv");
                exit(1);
            }

            IndexTime = GetLastIndexTimeForLot(LotNr,packedHedder.subname);

            sendall(socket,&IndexTime, sizeof(IndexTime));

        }
        else if (packedHedder.command == C_rSetIndexTime) {

            int Lotnr;
            if ((i=recv(socket, &LotNr, sizeof(LotNr),0)) == -1) {
                perror("recv");
                exit(1);
            }

            setLastIndexTimeForLot(LotNr,NULL,packedHedder.subname);

        }
        else if (packedHedder.command == C_rSendFile) {
            //skal mota en fil for lagring i reposetoryet
            //char FilePath[156];
            FILE *FILEHANDLER;
            char c;
            char opentype[2];
            //char *filblocbuff;
            //off_t fileBloks,filerest;

            if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) {
                perror("Cant recv lotnr");
                exit(1);
            }

            printf("lotNr %i\n",LotNr);


            //leser destinasjonelengden
            if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) {
                perror("Cant recv destLeng");
                exit(1);
            }

            if (destLeng > sizeof(dest)) {
                printf("dest filname is to long at %i\n",destLeng);
                exit(1);
            }

            //leser destinasjonene
            if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) {
                perror("Cant recv dest");
                exit(1);
            }

            printf("coping %s as length %i in to lot %i\n",dest,destLeng,LotNr);

            if ((i=recv(socket, &opentype, sizeof(char) +1,MSG_WAITALL)) == -1) {
                perror("Cant recv opentype");
                exit(1);
            }
            printf("opentype \"%s\"\n",opentype);


            //GetFilPathForLot(FilePath,LotNr,packedHedder.subname);

            //legger til filnavnet
            //strncat(FilePath,dest,sizeof(FilePath));

            //leser inn filstørelsen
            if ((i=recv(socket, &fileBloks, sizeof(fileBloks),MSG_WAITALL)) == -1) {
                perror("Cant recv fileBloks");
                exit(1);
            }

            if ((i=recv(socket, &filerest, sizeof(filerest),MSG_WAITALL)) == -1) {
                perror("Cant recv filerest");
                exit(1);
            }

            printf("fileBloks: %" PRId64 ", filerest: %" PRId64 "\n",fileBloks,filerest);

            //åpner filen
            if ((FILEHANDLER = lotOpenFileNoCasheByLotNr(LotNr,dest,opentype,'e',packedHedder.subname)) == NULL) {
                perror(dest);
            }

            filblocbuff = (char *)malloc(rNetTrabsferBlok);
            for(i=0; i < fileBloks; i++) {

                if ((n=recv(socket, filblocbuff, rNetTrabsferBlok,MSG_WAITALL)) == -1) {
                    perror("Cant recv dest");
                    exit(1);
                }

                fwrite(filblocbuff,sizeof(c),rNetTrabsferBlok,FILEHANDLER);
            }

            printf("did recv %i fileBloks\n",i);


            if ((n=recv(socket, filblocbuff, filerest,MSG_WAITALL)) == -1) {
                perror("Cant recv filerest");
                exit(1);
            }

            fwrite(filblocbuff,sizeof(c),filerest,FILEHANDLER);


            free(filblocbuff);

            fclose(FILEHANDLER);

            printf("\n");
        }
        else if (packedHedder.command == C_DIGetIp) {


            unsigned int DocID;
            struct DocumentIndexFormat DocumentIndexPost;

            //printf("got command C_DIGetIp\n");

            if ((i=recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
                perror("recv");
                exit(1);
            }

            //printf("DocID %u\n",DocID);

            DIRead(&DocumentIndexPost,DocID,packedHedder.subname);

            //printf("ipadress: %u\n",DocumentIndexPost.IPAddress);

            sendall(socket,&DocumentIndexPost.IPAddress, sizeof(DocumentIndexPost.IPAddress));


        }
        else if (packedHedder.command == C_anchorAdd) {
            size_t textlen;
            unsigned int DocID;
            char *text;

            printf("Add anchor....\n");
            if ((i = recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
                perror("recv");
                exit(1);
            } else if ((i = recv(socket, &textlen, sizeof(textlen), MSG_WAITALL)) == -1) {
                perror("recv(textlen)");
                exit(1);
            }
            text = malloc(textlen+1);
            text[textlen] = '\0';
            if ((i = recv(socket, text, textlen, MSG_WAITALL)) == -1) {
                perror("recv(text)");
                exit(1);
            }

            anchoraddnew(DocID, text, textlen, packedHedder.subname, NULL);
            printf("Text for %d: %s\n", DocID, text);

            free(text);
        }
        else if (packedHedder.command == C_anchorGet) {
            size_t len;
            char *text;
            int LotNr;
            unsigned int DocID;
            printf("Get anchor...\n");

            if ((i = recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
                perror("recv");
                exit(1);
            }
            printf("got DocID %u\n",DocID);
            LotNr = rLotForDOCid(DocID);
            printf("trying to read anchor\n");

            len = anchorRead(LotNr, packedHedder.subname, DocID, NULL, -1);
            printf("got anchor of length %i\n",len);

            sendall(socket, &len, sizeof(len));
            text = malloc(len+1);

            printf("readint it again\n");
            anchorRead(LotNr, packedHedder.subname, DocID, text, len+1);
            sendall(socket, text, len);
        }
        else if (packedHedder.command == C_readHTML) {
            /*
            unsigned int DocID;
            unsigned int len;
            char *text;
            char *acla, *acld;
            struct DocumentIndexFormat DocIndex;
            struct ReposetoryHeaderFormat ReposetoryHeader;

            if ((i = recv(socket, &DocID, sizeof(DocID), MSG_WAITALL)) == -1) {
            	perror("recv");
            	exit(1);
            }

            if ((i = recv(socket, &len, sizeof(len), MSG_WAITALL)) == -1) {

            	perror("recv(len)");
            	exit(1);
            }
            printf("len %u\n",len);
            text = malloc(len);

            if (text == NULL)
            	exit(1);

            DIRead(&DocIndex, DocID, packedHedder.subname);


            if (!rReadHtml(
            		text,
            		&len,
            		DocIndex.RepositoryPointer,
            		DocIndex.htmlSize,
            		DocID,
            		packedHedder.subname,
            		&ReposetoryHeader,
            		&acla,
            		&acld,
            		DocIndex.imageSize)) {
            	len = 0;
            	sendall(socket, &len, sizeof(len));
            } else {
            	++len; // \0
            	#ifdef DEBUG
            	printf("docID %u\n",DocID);
            	printf("Got: (len %i, real %i) ########################\n%s\n#####################\n", len, strlen(text), text);
            	#endif
            	sendall(socket, &len, sizeof(len));
            	sendall(socket, text, len);
            	sendall(socket, &ReposetoryHeader,sizeof(ReposetoryHeader));
            }

            free(text);
            */
        }
        /*
        runarb: 06 des 2007: vi har gåt bort fra denne metoden for nå, og bruker heller index over smb. Men tar vare på den da vi kan trenge den siden

        else if (packedHedder.command == C_urltodocid) {
        	char cmd;
        	int alloclen;
        	char *urlbuf;

        	if (urltodociddb == NULL) {
        		cmd = C_DOCID_NODB;
        		sendall(socket, &cmd, sizeof(cmd));
        		exit(1);
        	} else {
        		cmd = C_DOCID_READY;
        		sendall(socket, &cmd, sizeof(cmd));
        	}
        	cmd = C_DOCID_NEXT;

        	alloclen = 1024;
        	urlbuf = malloc(alloclen);

        	do {
        		unsigned int DocID;
        		size_t len;
        		if ((i = recv(socket, &cmd, sizeof(cmd), MSG_WAITALL)) == -1) {
        			err(1, "recv(cmd)");
        		}
        		if (cmd == C_DOCID_DONE)
        			break;

        		if ((i == recv(socket, &len, sizeof(len), MSG_WAITALL)) == -1) {
        			err(1, "recv(len)");
        		}
        		if (alloclen < len+1) {
        			free(urlbuf);
        			alloclen *= 2;
        			urlbuf = malloc(alloclen);
        		}
        		if ((i == recv(socket, urlbuf, len, MSG_WAITALL)) == -1) {
        			err(1, "recv(len)");
        		}
        		urlbuf[len] = '\0';

        		if (!getDocIDFromUrl(urltodociddb, urlbuf, &DocID)) {
        			cmd = C_DOCID_NOTFOUND;
        			sendall(socket, &cmd, sizeof(cmd));
        		} else {
        			cmd = C_DOCID_FOUND;
        			sendall(socket, &cmd, sizeof(cmd));
        			sendall(socket, &DocID, sizeof(DocID));
        		}
        	} while (1);

        	free(urlbuf);
        }
        */
        else {
            printf("unnown comand. %i\n", packedHedder.command);
        }
        //printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
    } //while

}
コード例 #5
0
ファイル: main.c プロジェクト: dateline/enterprise-search
int main (int argc, char *argv[]) {

        int lotNr;
	char lotServer[64];
	int pageCount;
	int i;

        unsigned int FiltetTime;
        unsigned int FileOffset;

        char htmlcompressdbuffer[524288];  //0.5 mb
        char imagebuffer[524288];  //0.5 mb
	
	int httpResponsCodes[nrOfHttpResponsCodes];

	struct ReposetoryHeaderFormat ReposetoryHeader;
	struct DocumentIndexFormat DocumentIndexPost;
	unsigned long int radress;
	FILE *revindexFilesHa[NrOfDataDirectorys];
	struct adultFormat adult;
	unsigned int lastIndexTime;
        if (argc < 2) {
                printf("Dette programet indekserer en lot. Gi det et lot nummer\n");
                exit(0);
        }

	for(i=0;i<nrOfHttpResponsCodes;i++) {
		httpResponsCodes[i] = 0;
	}

	lotNr = atoi(argv[1]);



	//find server based on lotnr
	lotlistLoad();
	lotlistGetServer(lotServer,lotNr);


	printf("vil index lot nr %i at %s\n",lotNr,lotServer);

                //finner siste indekseringstid
                lastIndexTime =  GetLastIndexTimeForLot(lotNr,subname);


                if(lastIndexTime == 0) {
                        printf("lastIndexTime is 0, skiping.\n");
                        exit(1);
                }

	//temp: må hente dette fra slot server eller fil
	FiltetTime = 0;
	FileOffset = 2140483648;
	//FileOffset = 1997015914;
	
	pageCount = 0;

		printf("Wil acess files localy\n");


		while (rGetNext(lotNr,&ReposetoryHeader,htmlcompressdbuffer,imagebuffer,&radress,FiltetTime,FileOffset,subname)) {


				DIRead(&DocumentIndexPost,ReposetoryHeader.DocID,subname);		

				DocumentIndexPost.RepositoryPointer = radress;

				//skiver til DocumentIndex
				DIWrite(&DocumentIndexPost,ReposetoryHeader.DocID,subname);
				

			++pageCount;
		
		}




	printf("indexed %i pages\n\n\n",pageCount);

	return 0;
}