/* Finner path for en lot fra docid */ void GetFilPathForLotByDocID(char *FilePath,int DocID,char subname[]) { int lot; lot = rLotForDOCid(DocID); GetFilPathForLot(FilePath,lot,subname); }
int bbdocument_deletecoll(char collection[]) { int LotNr; int i; char FilePath[512]; char IndexPath[512]; char DictionaryPath[512]; FILE *fh; debug("Deleting collection: \"%s\"\n",collection); LotNr = 1; while((fh =lotOpenFileNoCasheByLotNr(LotNr,"reposetory","r",'s',collection)) != NULL) { GetFilPathForLot(FilePath,LotNr,collection); fclose(fh); rrmdir(FilePath); ++LotNr; } for (i=0; i < 64; i++) { GetFilePathForIindex(FilePath,IndexPath,i,"Main","aa",collection); #ifdef DEBUG printf("FilePath: %s\nIndexPath: %s\n",FilePath,IndexPath); #endif if ((unlink(IndexPath) != 1) && (errno != ENOENT)) { //ENOENT=No such file or directory. Viser ikke feil hvis filen ikke fantes. Det er helt normalt perror("remove IndexPath"); } GetFilePathForIDictionary(FilePath,DictionaryPath,i,"Main","aa",collection); #ifdef DEBUG printf("FilePath: %s\nDictionaryPath: %s\n",FilePath,DictionaryPath); #endif if ((unlink(DictionaryPath) != 0) && (errno != ENOENT)) {//ENOENT=No such file or directory. Viser ikke feil hvis filen ikke fantes. Det er helt normalt perror("remove DictionaryPath"); } } //sletter i userToSubname.db struct userToSubnameDbFormat userToSubnameDb; if (!userToSubname_open(&userToSubnameDb,'w')) { printf("can't open users.db\n"); } else { userToSubname_deletecol(&userToSubnameDb,collection); userToSubname_close(&userToSubnameDb); } return 1; }
main (int argc, char *argv[]) { int LotNr; char lotPath[255]; struct ReposetoryHeaderFormat ReposetoryHeader; unsigned int radress; char htmlbuffer[524288]; char imagebuffer[524288]; char subname[maxSubnameLength]; struct udfileFormat udfilepost; FILE *FH; if (argc < 3) { printf("Error ingen lotnr spesifisert.\n\nEksempel på bruk for å lese lot 2:\n\trread 2 www\n"); exit(1); } LotNr = atoi(argv[1]); strncpy(subname,argv[2],sizeof(subname) -1); printf("lotnr %i\n",LotNr); GetFilPathForLot(lotPath,LotNr,subname); printf("Opning lot at: %s for %s\n",lotPath,subname); if ((FH = fopen(FilePath,"ab")) == NULL) { perror("tmpe file"); exit(1); } //loppergjenom alle int htmlbufferSize = sizeof(htmlbuffer); int count = 0; while (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,htmlbufferSize,imagebuffer,&radress,0,0,subname)) { if ((ReposetoryHeader.response == 200) && (ReposetoryHeader.imageSize == 0) && (ReposetoryHeader.htmlSize != 0)) { //printf("DocId: %i url: %s res %hi htmls %hi time %lu\n",ReposetoryHeader.DocID,ReposetoryHeader.url,ReposetoryHeader.response,ReposetoryHeader.htmlSize,ReposetoryHeader.time); //printf("################################\n%s##############################\n",htmlbuffer); strncpy(udfilepost.url,ReposetoryHeader.url,sizeof(udfilepost.url)); udfilepost.DocID = ReposetoryHeader.DocID; fwrite(&udfilepost,sizeof(udfilepost),1,FH); } ++count; } printf("Did analyse %i pages\n",count); fclose(FH); }
int lotOpenFileNoCache_direct(unsigned int DocID, char *resource, char *type, char lock, char *subname) { unsigned int LotNr = rLotForDOCid(DocID); int i; char FilePath[PATH_MAX]; char File [PATH_MAX]; int fd; printf("lotOpenFileNoCache_direct(subname: \"%s\", resource %s)\n",subname,resource); GetFilPathForLot(FilePath,LotNr,subname); strcpy(File,FilePath); strncat(File,resource,PATH_MAX); //var 128 #ifdef DEBUG printf("lotOpenFileNoCasheByLotNr: opening file \"%s\" for %s\n",File,type); #endif //hvis dette er lesing så hjelper det ikke og prøve å opprette path. Filen vil fortsatt ikke finnes if ((strcmp(type,"rb") == 0) || (strcmp(type,"r") == 0)) { if ((fd = open64(File, O_RDONLY|O_DIRECT|O_LARGEFILE)) == -1) { warn("open64: %d", fd); #ifdef DEBUG perror(File); #endif return -1; } } else { errx(1, "We can only open this for reading right now"); } #ifdef DEBUG printf("lotOpenFile: tryint to obtain lock \"%c\"\n",lock); #endif //honterer låsning if (lock == 'e') { //skal vi ha flock64() her ? flock(fd, LOCK_EX); } else if (lock == 's') { flock(fd, LOCK_SH); } #ifdef DEBUG printf("lotOpenFile: lock obtained\n"); #endif #ifdef DEBUG printf("lotOpenFileNoCasheByLotNr: finished\n"); #endif return fd; }
void makeLotPath(int lotNr,char folder[],char subname[]) { char path[PATH_MAX]; GetFilPathForLot(path,lotNr,subname); strcat(path,"/"); strcat(path,folder); makePath(path); }
main (int argc, char *argv[]) { int LotNr; char lotPath[255]; int rank; struct ReposetoryHeaderFormat ReposetoryHeader; unsigned int radress; char htmlbuffer[524288]; char imagebuffer[524288]; char *acl; if (argc < 5) { printf("Error ingen lotnr spesifisert.\n\nEksempel på bruk for å lese lot 2:\n\trread 2 www servername minrank\n"); exit(1); } LotNr = atoi(argv[1]); char *subname = argv[2]; char *servername = argv[3]; int minrank = atoi(argv[4]); printf("lotnr %i\n",LotNr); GetFilPathForLot(lotPath,LotNr,subname); printf("Opning lot at: %s for %s\n",lotPath,subname); popopenMemArray(servername,"www"); //loppergjenom alle // int rGetNext (unsigned int LotNr,struct ReposetoryHeaderFormat *ReposetoryHeader, char htmlbuffer[], // int htmlbufferSize, char imagebuffer[], unsigned int *radress, // unsigned int FilterTime, unsigned int FileOffset, char subname[]); while (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,sizeof(htmlbuffer),imagebuffer,&radress,0,0,subname,&acl)) { rank = popRankForDocIDMemArray(ReposetoryHeader.DocID); if (rank >= minrank) { //printf("DocId: %i url: %s res %hi htmls %hi time %lu, rank %i\n",ReposetoryHeader.DocID,ReposetoryHeader.url,ReposetoryHeader.response,ReposetoryHeader.htmlSize,ReposetoryHeader.time,rank); printf("%s\n",ReposetoryHeader.url); } //printf("################################\n%s##############################\n",htmlbuffer); } }
//gir ful path for et bilde fra DocID void GetFilPathForThumbnaleByDocID(char *FileName,int DocID,char subname[]) { int LotNr; int ImageBucket; ImageBucket = fmod(DocID,512); //finner path LotNr = rLotForDOCid(DocID); GetFilPathForLot(FileName,LotNr,subname); sprintf(FileName,"%simages/%i/%i.jpg",FileName,ImageBucket,DocID); }
void cache_fresh_lot_collection(void) { DIR *colls; char *coll; char path[2048]; size_t len; int i; /* We only look at the first 5 lots */ if ((colls = listAllColl_start()) == NULL) { bblog(ERROR, "Can't listAllColl_start()"); return; } while ((coll = listAllColl_next(colls))) { for (i = 1; i < 6; i++) { DIR *dirp; struct dirent *de; GetFilPathForLot(path, i, coll); len = strlen(path); dirp = opendir(path); if (dirp == NULL) continue; while ((de = readdir(dirp))) { int fd; int dw; if (de->d_name[0] == '.') continue; sprintf(path+len, "/%s", de->d_name); bblog(DEBUGINFO, "Found file: %s", path); fd = open(path, O_RDONLY); if (fd == -1) continue; read(fd, &dw, sizeof(dw)); close(fd); } closedir(dirp); } } listAllColl_close(colls); }
int lotHasSufficientSpace(int lot, int needSpace,char subname[]) { char FilePath[512]; int i; GetFilPathForLot(FilePath,lot,subname); //må lage, slik at vi er 100% sikker på at vi har noe å teste mot #ifdef DEBUG printf("will hav to make lot path sow we can test for space\n"); #endif makePath(FilePath); i=HasSufficientSpace(FilePath,needSpace); return i; }
//sjeker om det fins en DocumentIndex fro denne loten int DIHaveIndex (int lotNr,char subname[]) { char FilePath[512]; FILE *FH; GetFilPathForLot(FilePath,lotNr,subname); strncat(FilePath,"DocumentIndex",sizeof(FilePath)); if ((FH = fopen(FilePath,"r")) == NULL) { return 0; } else { fclose(FH); return 1; } }
int main(int argc, char **argv) { FILE *fp; struct redirects redir; char *subname; int LotNr; char filename[1024]; char *text; size_t len; if (argc < 2) err(1, "Usage: ./readtest lotnr subname"); LotNr = atoi(argv[1]); subname = argv[2]; GetFilPathForLot(filename, LotNr, subname); strcat(filename, "redirmap"); if ((fp = fopen(filename, "r")) == NULL) err(1, "fopen(%s)", filename); len = 1048576 * 5; //5MB if ((text = malloc(len)) == NULL) { perror("malloc"); exit(1); } while (fread(&redir, sizeof(redir), 1, fp) == 1) { int len2; printf("%u => %u (reason: %hu)\n", redir.DocID, redir.redirectTo, redir.response); len2 = readHTMLNET(subname, redir.redirectTo, text, len); printf("fiii: %d\n", len2); if (text[0] != '\0') addResource(LotNr, subname, redir.DocID, text, len2); // printf("And got html: ''%s''\n", text); } free(text); fclose(fp); return 0; }
void popopenMemArray_oneLot(char subname[], int i) { FILE *FH; char LotFile[128]; int branksize; GetFilPathForLot(LotFile,i,subname); strcat(LotFile,"Brank"); // prøver å opne if ( (FH = fopen(LotFile,"rb")) == NULL ) { perror(LotFile); popMemArray[i] = 0; } else { #ifdef DEBUG printf("loaded lot %i\n",i); #endif branksize = sizeof(unsigned char) * NrofDocIDsInLot; if ((popMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) { printf("malloc eror for lot %i\n",i); perror("malloc"); exit(1); } fread(popMemArray[i],branksize,1,FH); //debug: viser alle rankene vi laster /* int y; for(y=0;y<NrofDocIDsInLot;y++) { printf("DocID %i, rank %u. i:%i, y:%i\n",(y + LotDocIDOfset(i)),(unsigned int)popMemArray[i][y],i,y); } */ fclose(FH); } }
void revindexFilesOpenLocal(FILE *revindexFilesHa[],int lotNr) { int i; char lotPath[128]; char revfile[128]; GetFilPathForLot(lotPath,lotNr); //oppretter path sprintf(revfile,"%srevindex/Main/",lotPath); makePath(revfile); for(i=0;i<NrOfDataDirectorys;i++) { sprintf(revfile,"%srevindex/Main/%i.txt",lotPath,i); //printf("opning revfile %s\n",revfile); if ((revindexFilesHa[i] = fopen(revfile,"wb")) == NULL) { perror(revfile); exit(1); } } }
int main(int argc, char **argv) { int LotNr; char lotPath[255]; struct DocumentIndexFormat docindex; unsigned int DocID; char text[40]; unsigned int radress; if (argc < 3) errx(1, "Usage: ./DIread lotNr subname"); #if 0 if (argc > 3) anchoraddnew(125500001, "eirik", 5, "www", NULL); #endif LotNr = atoi(argv[1]); char *subname = argv[2]; printf("lotnr %i\n",LotNr); GetFilPathForLot(lotPath,LotNr,subname); printf("Opning lot at: %s for %s\n",lotPath,subname); #if 1 //loppergjenom alle while (DIGetNext(&docindex, LotNr, &DocID, subname)) { printf("DocID: %d\n", DocID); if (anchorRead(LotNr, subname, DocID, text, sizeof(text))) printf("\tAnchor text: '%s'\n", text); } #endif return 0; }
void popopenMemArray2(char subname[], char rankfile[]) { FILE *FH; int i, y, z; int LocalLots; char LotFile[128]; int branksize; struct stat inode; off_t totsize = 0; int locklimit; //finner grensen på antal sider vi kan låse i minne struct rlimit rlim; if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) { printf("Warning: Cannot raise RLIMIT_MEMLOCK limits."); locklimit = 0; } else { locklimit = rlim.rlim_cur; } //aloker minne for rank for de forskjelige lottene LocalLots=0; for (i=0;i<maxLots;i++) { //sjekekr om dette er en lokal lot popMemArray[i] = 0; GetFilPathForLot(LotFile,i,subname); strcat(LotFile,rankfile); // prøver å opne if ( (FH = fopen(LotFile,"rb")) == NULL ) { #ifdef DEBUG perror(LotFile); #endif continue; } fstat(fileno(FH),&inode); if (inode.st_size == 0) { printf("file is emty\n"); continue; } #ifdef DEBUG printf("popopenMemArray2: loaded lot %i\n",i); #endif branksize = sizeof(unsigned char) * NrofDocIDsInLot; printf("branksize offset: %i\n",branksize % getpagesize()); #ifdef MMAP_POP //vi må aligne dette med pagesize //branksize += branksize % getpagesize(); if (inode.st_size != branksize) { fprintf(stderr,"popopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize); continue; } //MAP_LOCKED (since Linux 2.5.37) // Lock the pages of the mapped region into memory in the manner of mlock(). This flag is ignored in older kernels. if ((popMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == MAP_FAILED) { fprintf(stderr,"popopenMemArray: can't mmap for lot %i\n",i); perror("popopenMemArray2 mmap"); } //hvis vi kan låse uendelig med minne gjør vi det if (locklimit == -1) { //laster all rankerings dataen fra minne, slik ad det går fort å leste den inn siden z = 0; for(y=0;y<NrofDocIDsInLot;y++) { z += popMemArray[i][y]; } //låser minne if (mlock(popMemArray[i],branksize) != 0) { perror("mlock"); } } #else if ((popMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) { printf("malloc eror for lot %i\n",i); perror("malloc"); exit(1); } fread(popMemArray[i],branksize,1,FH); #endif //debug: viser alle rankene vi laster /* for(y=0;y<branksize;y++) { printf("DocID %i, rank %i\n",y,popMemArray[i][y]); } */ totsize += branksize; fclose(FH); ++LocalLots; } printf("popopenMemArray: have %i local lots\n",LocalLots); printf("popopenMemArray: loaded a total of %"PRId64" bytes\n",totsize); }
void connectHandler(int socket) { struct packedHedderFormat packedHedder; int isAuthenticated = 0; char tkeyForTest[32]; int i,n; int intrespons; int count = 0; container *attrkeys = NULL; #ifdef DEBUG_TIME struct timeval start_time, end_time; struct timeval tot_start_time, tot_end_time; gettimeofday(&tot_start_time, NULL); #endif ionice_benice(); while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) { #ifdef DEBUG printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command); #endif packedHedder.size = packedHedder.size - sizeof(packedHedder); if (attrkeys == NULL) { attrkeys = ropen(); } if (packedHedder.command == bbc_askToAuthenticate) { if ((i=recv(socket, tkeyForTest, sizeof(tkeyForTest),MSG_WAITALL)) == -1) { perror("Cant read tkeyForTest"); exit(1); } if (1) { printf("authenticated\n"); intrespons = bbc_authenticate_ok; bbdocument_init(NULL); isAuthenticated = 1; } else { printf("authenticate faild\n"); intrespons = bbc_authenticate_feiled; } if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) { perror("Cant recv filerest"); exit(1); } } else { if (!isAuthenticated) { printf("user not autentikated\n"); exit(1); } if (packedHedder.command == bbc_docadd) { #ifdef DEBUG printf("bbc_docadd\n"); #endif char *subname,*documenturi,*documenttype,*document,*acl_allow,*acl_denied,*title,*doctype; char *attributes; int dokument_size; unsigned int lastmodified; #ifdef DEBUG_TIME gettimeofday(&start_time, NULL); #endif //subname if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } subname = malloc(intrespons +1); if ((i=recvall(socket, subname, intrespons)) == 0) { perror("Cant read subname"); exit(1); } //documenturi if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } documenturi = malloc(intrespons +1); if ((i=recvall(socket, documenturi, intrespons)) == 0) { perror("Cant read documenturi"); exit(1); } //documenttype if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } documenttype = malloc(intrespons +1); if ((i=recvall(socket, documenttype, intrespons)) == 0) { perror("Cant read documenttype"); exit(1); } //document //dokument_size if ((i=recvall(socket, &dokument_size, sizeof(dokument_size))) == 0) { perror("Cant read dokument_size"); exit(1); } document = malloc(dokument_size +1); if (dokument_size == 0) { document[0] = '\0'; } else { if ((i=recvall(socket, document, dokument_size)) == 0) { fprintf(stderr,"Can't read document of size %i\n",dokument_size); perror("recvall"); exit(1); } } //lastmodified if ((i=recvall(socket, &lastmodified, sizeof(lastmodified))) == 0) { perror("Cant read lastmodified"); exit(1); } //acl_allow if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } acl_allow = malloc(intrespons +1); if ((i=recvall(socket, acl_allow, intrespons)) == 0) { perror("Cant read acl_allow"); exit(1); } //acl_denied if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } acl_denied = malloc(intrespons +1); if ((i=recvall(socket, acl_denied, intrespons)) == 0) { perror("Cant read acl_denied"); exit(1); } //title if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } title = malloc(intrespons +1); if ((i=recvall(socket, title, intrespons)) == 0) { perror("Cant read title"); exit(1); } //doctype if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } doctype = malloc(intrespons +1); if ((i=recvall(socket, doctype, intrespons)) == 0) { perror("Cant read doctype"); exit(1); } // Attribute list if ((i = recvall(socket, &intrespons, sizeof(intrespons))) == 0) err(1, "Can't receive attribute list len"); attributes = malloc(intrespons +1); if ((i=recvall(socket, attributes, intrespons)) == 0) err(1, "Can't receive attribute list"); #ifdef DEBUG_TIME gettimeofday(&end_time, NULL); printf("Time debug: bbdn_docadd recv data time: %f\n",getTimeDifference(&start_time, &end_time)); #endif printf("\n"); printf("########################################################\n"); printf("Url: %s\n",documenturi); printf("got subname \"%s\": title \"%s\". Nr %i, dokument_size %i attrib: %s\n",subname,title,count,dokument_size, attributes); printf("########################################################\n"); printf("calling bbdocument_add():\n"); #ifdef DEBUG_TIME gettimeofday(&start_time, NULL); #endif intrespons = bbdocument_add(subname,documenturi,documenttype,document,dokument_size,lastmodified,acl_allow,acl_denied,title,doctype, attributes, attrkeys); printf(":bbdocument_add end\n"); printf("########################################################\n"); #ifdef DEBUG_TIME gettimeofday(&end_time, NULL); printf("Time debug: bbdn_docadd runing bbdocument_add() time: %f\n",getTimeDifference(&start_time, &end_time)); #endif free(subname); free(documenturi); free(documenttype); free(document); free(acl_allow); free(acl_denied); free(title); free(doctype); free(attributes); // send status if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) { perror("Cant recv filerest"); exit(1); } } else if (packedHedder.command == bbc_opencollection) { char *subname; char path[PATH_MAX]; printf("open collection\n"); if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) err(1, "Cant read intrespons"); subname = malloc(intrespons +1); if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) err(1, "Cant read subname"); GetFilPathForLot(path, 1, subname); strcat(path, "fullyCrawled"); unlink(path); free(subname); } else if (packedHedder.command == bbc_closecollection) { printf("closecollection\n"); char *subname; //subname if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) { perror("Cant read intrespons"); exit(1); } subname = malloc(intrespons +1); if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) { perror("Cant read subname"); exit(1); } bbdocument_close(attrkeys); attrkeys = NULL; //toDo må bruke subname, og C ikke perl her printf("cleanin lots start\n"); char command[PATH_MAX]; snprintf(command,sizeof(command),"perl %s -l -s \"%s\"",bfile("perl/cleanLots.pl"),subname); printf("running \"%s\"\n",command); intrespons = system(command); printf("cleanin lots end\n"); // legger subnamet til listen over ventene subnavn, og huper searchd. lot_recache_collection(subname); /* We are done crawling */ { int fd = lotOpenFileNoCasheByLotNrl(1, "fullyCrawled", ">>", '\0', subname); if (fd == -1) { warn("Unable to write fullyCrawled file"); } else { close(fd); } } free(subname); if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) { perror("Cant recv filerest"); exit(1); } } else if (packedHedder.command == bbc_deleteuri) { printf("deleteuri\n"); char *subname, *uri; //subname if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) { perror("Cant read intrespons"); exit(1); } subname = malloc(intrespons +1); if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) { perror("Cant read subname"); exit(1); } subname[intrespons] = '\0'; if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) { perror("Cant read intrespons"); exit(1); } uri = malloc(intrespons +1); if ((i=recv(socket, uri, intrespons,MSG_WAITALL)) == -1) { perror("Cant read uri"); exit(1); } uri[intrespons] = '\0'; printf("going to delete: %s from %s\n", uri, subname); /* Add docid to the gced file */ { FILE *fh; unsigned int DocID, lastmodified; unsigned int lotNr; int err = 0; if (uriindex_get(uri, &DocID, &lastmodified, subname) == 0) { fprintf(stderr,"Unable to get uri info. uri=\"%s\",subname=\"%s\".",uri,subname); perror("Unable to get uri info"); err++; } if (!err) { lotNr = rLotForDOCid(DocID); if ((fh = lotOpenFileNoCasheByLotNr(lotNr,"gced","a", 'e',subname)) == NULL) { perror("can't open gced file"); err++; } else { fwrite(&DocID, sizeof(DocID), 1, fh); fclose(fh); } } if (!err) { struct reformat *re; if((re = reopen(rLotForDOCid(DocID), sizeof(struct DocumentIndexFormat), "DocumentIndex", subname, RE_HAVE_4_BYTES_VERSION_PREFIX)) == NULL) { perror("can't reopen()"); err++; } else { DIS_delete(RE_DocumentIndex(re, DocID)); reclose(re); } } //markerer at den er skitten if (!err) { FILE *dirtfh; dirtfh = lotOpenFileNoCashe(DocID,"dirty","ab",'e',subname); fwrite("1",1,1,dirtfh); fclose(dirtfh); } if (err == 0) bbdocument_delete(uri, subname); } free(subname); intrespons = 1; // Always return ok for now if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) { perror("Cant recv filerest"); exit(1); } } else if (packedHedder.command == bbc_deletecollection) { printf("deletecollection\n"); char *subname, *uri; //subname if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) { perror("Cant read intrespons"); exit(1); } subname = malloc(intrespons +1); if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) { perror("Cant read subname"); exit(1); } subname[intrespons] = '\0'; printf("going to delete collection: %s\n", subname); intrespons = bbdocument_deletecoll(subname); if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) { perror("Cant recv filerest"); exit(1); } free(subname); } else if (packedHedder.command == bbc_addwhisper) { whisper_t add; char *subname; if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) err(1, "Cant read intrespons"); subname = malloc(intrespons+1); if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) { perror("Cant read subname"); exit(1); } subname[intrespons] = '\0'; if ((i=recv(socket, &add, sizeof(add),MSG_WAITALL)) == -1) err(1, "Cant read add whisper"); gcwhisper_write(subname, add); free(subname); } else if (packedHedder.command == bbc_HasSufficientSpace) { char *subname; //subname if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) { perror("Cant read intrespons"); exit(1); } subname = malloc(intrespons +1); if ((i=recvall(socket, subname, intrespons)) == 0) { perror("Cant read subname"); exit(1); } // tester bare i lot 1 her. Må også sjekke andre loter når vi begynner å støtte frlere disker på ES. intrespons = lotHasSufficientSpace(1, 4096, subname); if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) { perror("Cant recv filerest"); exit(1); } printf("~Asked for HasSufficientSpace for subname \"%s\". Returnerer %d\n",subname, intrespons); free(subname); } else { printf("unnown comand. %i\n", packedHedder.command); } } ++count; // #ifdef DEBUG_BREAK_AFTER // if (count >= DEBUG_BREAK_AFTER) { // printf("exeting after %i docoments\n",count); // exit(1); // } // #endif } #ifdef DEBUG_TIME gettimeofday(&tot_end_time, NULL); printf("Time debug: bbdn total time time: %f\n",getTimeDifference(&tot_start_time, &tot_end_time)); #endif }
void connectHandler(int socket) { struct packedHedderFormat packedHedder; int i,n; int LotNr; char lotPath[512]; char buf[100]; unsigned int FilterTime; int filnamelen; FILE *FH; struct stat inode; // lager en struktur for fstat å returnere. off_t filesize; char c; struct DocumentIndexFormat DocumentIndexPost; int DocID; struct ReposetoryHeaderFormat ReposetoryHeader; unsigned int radress; char htmlbuffer[524288]; int destLeng; char dest[512]; off_t fileBloks,filerest; char *filblocbuff; //while ((i=read(socket, &packedHedder, sizeof(struct packedHedderFormat))) > 0) { while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) { //printf("command: %i\n",packedHedder.command); //printf("i er %i\n",i); printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command); //printf("subname: %s\n",packedHedder.subname); //lar size reflektere hva som er igjen av pakken packedHedder.size = packedHedder.size - sizeof(packedHedder); if (packedHedder.command == C_rmkdir) { printf("C_rmkdir\n"); //leser data. Det skal væren en int som sier hvilken lot vi vil ha if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) { perror("Cant read lotnr"); exit(1); } //leser destinasjonelengden if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) { perror("Cant read destLeng"); exit(1); } if (destLeng > sizeof(dest)) { printf("dest filname is to long at %i\n",destLeng); exit(1); } //leser destinasjonene if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) { perror("Cant read dest"); exit(1); } GetFilPathForLot(lotPath,LotNr,packedHedder.subname); sprintf(lotPath,"%s%s",lotPath,dest); printf("mkdir %s\n",lotPath); makePath(lotPath); printf("~C_rmkdir\n"); } else if (packedHedder.command == C_rComand) { //leser data. Det skal væren en int som sier hvilken lot vi vil ha if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) { perror("Cant read lotnr"); exit(1); } //leser destinasjonelengden if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) { perror("Cant read destLeng"); exit(1); } if (destLeng > sizeof(dest)) { printf("dest filname is to long at %i\n",destLeng); exit(1); } //leser destinasjonene if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) { perror("Cant read dest"); exit(1); } printf("run command %s\n",dest); system(dest); } else if (packedHedder.command == C_getLotToIndex) { printf("fikk C_getLotToIndex\n"); int dirty; if ((i=recv(socket, &dirty, sizeof(dirty),MSG_WAITALL)) == -1) { perror("Cant read dirty"); exit(1); } printf("dirty: %i\n",dirty); LotNr = findLotToIndex(packedHedder.subname,dirty); printf("sending respons\n"); sendall(socket,&LotNr, sizeof(LotNr)); } else if (packedHedder.command == C_getlotHasSufficientSpace) { printf("fikk C_getLotToIndex\n"); int needSpace; int response; if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) { perror("Cant read lotnr"); exit(1); } if ((i=recv(socket, &needSpace, sizeof(needSpace),MSG_WAITALL)) == -1) { perror("Cant read dirty"); exit(1); } printf("needSpace: %i, LotNr %i\n",needSpace,LotNr); response = lotHasSufficientSpace(LotNr, needSpace, packedHedder.subname); printf("sending respons\n"); sendall(socket,&response, sizeof(response)); } else if (packedHedder.command == C_rGetSize) { printf("fikk C_rGetSize\n"); //leser data. Det skal væren en int som sier hvilken lot vi vil ha if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) { perror("Cant read lotnr"); exit(1); } if ((i=read(socket, &filnamelen, sizeof(filnamelen))) == -1) { perror("Cant read filnamelen"); exit(1); } if (filnamelen > sizeof(buf)) { printf("filname to long\n"); }; if ((i=read(socket, buf, filnamelen)) == -1) { perror("Cant read filnamelen"); exit(1); } printf("filname %s\n",buf); if ((FH = lotOpenFileNoCasheByLotNr(LotNr,buf,"rb",'s',packedHedder.subname)) == NULL) { perror(buf); //sending that he fil is emty fileBloks = 0; sendall(socket,&fileBloks, sizeof(fileBloks)); } else { //finner og sender il størelse fstat(fileno(FH),&inode); //filesize = inode.st_size; //sendall(socket,&filesize, sizeof(filesize)); fileBloks = inode.st_size; printf("size is %" PRId64 "\n",fileBloks); sendall(socket,&fileBloks, sizeof(fileBloks)); fclose(FH); } } else if (packedHedder.command == C_rGetFile) { printf("fikk C_rGetFile\n"); //leser data. Det skal væren en int som sier hvilken lot vi vil ha if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) { perror("Cant read lotnr"); exit(1); } if ((i=read(socket, &filnamelen, sizeof(filnamelen))) == -1) { perror("Cant read filnamelen"); exit(1); } if (filnamelen > sizeof(buf)) { printf("filname to long\n"); }; if ((i=read(socket, buf, filnamelen)) == -1) { perror("Cant read filnamelen"); exit(1); } printf("filname %s\n",buf); if ((FH = lotOpenFileNoCasheByLotNr(LotNr,buf,"rb",'s',packedHedder.subname)) == NULL) { perror(buf); //sending that the fil is emty fileBloks = 0; filerest = 0; sendall(socket,&fileBloks, sizeof(fileBloks)); sendall(socket,&filerest, sizeof(filerest)); } else { //finner og sender fil størelse fstat(fileno(FH),&inode); //filesize = inode.st_size; //sendall(socket,&filesize, sizeof(filesize)); fileBloks = (int)floor(inode.st_size / rNetTrabsferBlok); filerest = inode.st_size - (fileBloks * rNetTrabsferBlok); sendall(socket,&fileBloks, sizeof(fileBloks)); sendall(socket,&filerest, sizeof(filerest)); printf("sending fil. fileBloks %"PRId64", filerest %"PRId64"\n",fileBloks,filerest); filblocbuff = (char *)malloc(rNetTrabsferBlok); for(i=0; i < fileBloks; i++) { //fread(filblocbuff,sizeof(c),rNetTrabsferBlok,FH); //fread_all(const void *buf, size_t size, FILE *stream) fread_all(filblocbuff,rNetTrabsferBlok,FH, 4096); if ((n=sendall(socket, filblocbuff, rNetTrabsferBlok)) == -1) { perror("Cant recv dest"); exit(1); } } printf("did recv %i fileBloks\n",i); fread(filblocbuff,sizeof(c),filerest,FH); if ((n=sendall(socket, filblocbuff, filerest)) == -1) { perror("Cant recv filerest"); exit(1); } free(filblocbuff); /* for (i=0;i<filesize;i++) { fread(&c,sizeof(char),1,FH); send(socket, &c, sizeof(char), 0); //printf("%i\n",(int)c); } */ printf("send file end\n"); fclose(FH); } } else if (packedHedder.command == C_rGetNext) { printf("fikk C_rGetNext\n"); printf("støttes ikke lengere"); exit(1); /* //leser data. Det skal væren en unigned int som sier hvilken lot vi vil ha //har deklarert den som int her ??? if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) { perror("Cant read lotnr"); exit(1); } printf("leser FilterTime\n"); //leser filtertime if ((i=read(socket, &FilterTime, sizeof(FilterTime))) == -1) { perror("Cant read lotnr"); exit(1); } printf("lotnr %i FilterTime %u\n",LotNr,FilterTime); //henter inn data om den lotten if (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,NULL,&radress,FilterTime,0)) { //printf("DocId: %i url: %s\n",ReposetoryHeader.DocID,ReposetoryHeader.url); //sender pakke hedder sendpacked(socket,C_rLotData,PROTOCOLVERSION, ReposetoryHeader.htmlSize + sizeof(ReposetoryHeader) +sizeof(radress), NULL,packedHedder.subname); //sennder ReposetoryHeader'en sendall(socket,&ReposetoryHeader, sizeof(ReposetoryHeader)); //sender htmlen sendall(socket,&htmlbuffer, ReposetoryHeader.htmlSize); //sender adressen sendall(socket,&radress,sizeof(radress)); //printf("data sent\n"); //printf("rGetNext: %i\n",ReposetoryHeader.DocID); } else { sendpacked(socket,C_rEOF,PROTOCOLVERSION, 0, NULL,packedHedder.subname); printf("ferdig\n"); } */ } else if (packedHedder.command == C_DIWrite) { if ((i=recv(socket, &DocumentIndexPost, sizeof(struct DocumentIndexFormat),MSG_WAITALL)) == -1) { perror("recv"); exit(1); } if ((i=recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) { perror("recv"); exit(1); } DIWrite(&DocumentIndexPost,DocID,packedHedder.subname, NULL); //printf("DIWrite: %i\n",DocID); } else if (packedHedder.command == C_DIRead) { int DocID; struct DocumentIndexFormat DocumentIndexPost; printf("got commane C_DIRead. sise %i hsize %i ds %i\n",packedHedder.size, sizeof(packedHedder), sizeof(DocID)); if ((i=recv(socket, &DocID, sizeof(DocID),0)) == -1) { perror("recv"); exit(1); } //printf("DocID %i\n",DocID); //leser inn datan //int DIRead (struct DocumentIndexFormat *DocumentIndexPost, int DocID); DIRead(&DocumentIndexPost,DocID,packedHedder.subname); sendall(socket,&DocumentIndexPost, sizeof(struct DocumentIndexFormat)); } else if (packedHedder.command == C_rGetIndexTime) { int Lotnr; unsigned int IndexTime; if ((i=recv(socket, &LotNr, sizeof(LotNr),0)) == -1) { perror("recv"); exit(1); } IndexTime = GetLastIndexTimeForLot(LotNr,packedHedder.subname); sendall(socket,&IndexTime, sizeof(IndexTime)); } else if (packedHedder.command == C_rSetIndexTime) { int Lotnr; if ((i=recv(socket, &LotNr, sizeof(LotNr),0)) == -1) { perror("recv"); exit(1); } setLastIndexTimeForLot(LotNr,NULL,packedHedder.subname); } else if (packedHedder.command == C_rSendFile) { //skal mota en fil for lagring i reposetoryet //char FilePath[156]; FILE *FILEHANDLER; char c; char opentype[2]; //char *filblocbuff; //off_t fileBloks,filerest; if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) { perror("Cant recv lotnr"); exit(1); } printf("lotNr %i\n",LotNr); //leser destinasjonelengden if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) { perror("Cant recv destLeng"); exit(1); } if (destLeng > sizeof(dest)) { printf("dest filname is to long at %i\n",destLeng); exit(1); } //leser destinasjonene if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) { perror("Cant recv dest"); exit(1); } printf("coping %s as length %i in to lot %i\n",dest,destLeng,LotNr); if ((i=recv(socket, &opentype, sizeof(char) +1,MSG_WAITALL)) == -1) { perror("Cant recv opentype"); exit(1); } printf("opentype \"%s\"\n",opentype); //GetFilPathForLot(FilePath,LotNr,packedHedder.subname); //legger til filnavnet //strncat(FilePath,dest,sizeof(FilePath)); //leser inn filstørelsen if ((i=recv(socket, &fileBloks, sizeof(fileBloks),MSG_WAITALL)) == -1) { perror("Cant recv fileBloks"); exit(1); } if ((i=recv(socket, &filerest, sizeof(filerest),MSG_WAITALL)) == -1) { perror("Cant recv filerest"); exit(1); } printf("fileBloks: %" PRId64 ", filerest: %" PRId64 "\n",fileBloks,filerest); //åpner filen if ((FILEHANDLER = lotOpenFileNoCasheByLotNr(LotNr,dest,opentype,'e',packedHedder.subname)) == NULL) { perror(dest); } filblocbuff = (char *)malloc(rNetTrabsferBlok); for(i=0; i < fileBloks; i++) { if ((n=recv(socket, filblocbuff, rNetTrabsferBlok,MSG_WAITALL)) == -1) { perror("Cant recv dest"); exit(1); } fwrite(filblocbuff,sizeof(c),rNetTrabsferBlok,FILEHANDLER); } printf("did recv %i fileBloks\n",i); if ((n=recv(socket, filblocbuff, filerest,MSG_WAITALL)) == -1) { perror("Cant recv filerest"); exit(1); } fwrite(filblocbuff,sizeof(c),filerest,FILEHANDLER); free(filblocbuff); fclose(FILEHANDLER); printf("\n"); } else if (packedHedder.command == C_DIGetIp) { unsigned int DocID; struct DocumentIndexFormat DocumentIndexPost; //printf("got command C_DIGetIp\n"); if ((i=recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) { perror("recv"); exit(1); } //printf("DocID %u\n",DocID); DIRead(&DocumentIndexPost,DocID,packedHedder.subname); //printf("ipadress: %u\n",DocumentIndexPost.IPAddress); sendall(socket,&DocumentIndexPost.IPAddress, sizeof(DocumentIndexPost.IPAddress)); } else if (packedHedder.command == C_anchorAdd) { size_t textlen; unsigned int DocID; char *text; printf("Add anchor....\n"); if ((i = recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) { perror("recv"); exit(1); } else if ((i = recv(socket, &textlen, sizeof(textlen), MSG_WAITALL)) == -1) { perror("recv(textlen)"); exit(1); } text = malloc(textlen+1); text[textlen] = '\0'; if ((i = recv(socket, text, textlen, MSG_WAITALL)) == -1) { perror("recv(text)"); exit(1); } anchoraddnew(DocID, text, textlen, packedHedder.subname, NULL); printf("Text for %d: %s\n", DocID, text); free(text); } else if (packedHedder.command == C_anchorGet) { size_t len; char *text; int LotNr; unsigned int DocID; printf("Get anchor...\n"); if ((i = recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) { perror("recv"); exit(1); } printf("got DocID %u\n",DocID); LotNr = rLotForDOCid(DocID); printf("trying to read anchor\n"); len = anchorRead(LotNr, packedHedder.subname, DocID, NULL, -1); printf("got anchor of length %i\n",len); sendall(socket, &len, sizeof(len)); text = malloc(len+1); printf("readint it again\n"); anchorRead(LotNr, packedHedder.subname, DocID, text, len+1); sendall(socket, text, len); } else if (packedHedder.command == C_readHTML) { /* unsigned int DocID; unsigned int len; char *text; char *acla, *acld; struct DocumentIndexFormat DocIndex; struct ReposetoryHeaderFormat ReposetoryHeader; if ((i = recv(socket, &DocID, sizeof(DocID), MSG_WAITALL)) == -1) { perror("recv"); exit(1); } if ((i = recv(socket, &len, sizeof(len), MSG_WAITALL)) == -1) { perror("recv(len)"); exit(1); } printf("len %u\n",len); text = malloc(len); if (text == NULL) exit(1); DIRead(&DocIndex, DocID, packedHedder.subname); if (!rReadHtml( text, &len, DocIndex.RepositoryPointer, DocIndex.htmlSize, DocID, packedHedder.subname, &ReposetoryHeader, &acla, &acld, DocIndex.imageSize)) { len = 0; sendall(socket, &len, sizeof(len)); } else { ++len; // \0 #ifdef DEBUG printf("docID %u\n",DocID); printf("Got: (len %i, real %i) ########################\n%s\n#####################\n", len, strlen(text), text); #endif sendall(socket, &len, sizeof(len)); sendall(socket, text, len); sendall(socket, &ReposetoryHeader,sizeof(ReposetoryHeader)); } free(text); */ } /* runarb: 06 des 2007: vi har gåt bort fra denne metoden for nå, og bruker heller index over smb. Men tar vare på den da vi kan trenge den siden else if (packedHedder.command == C_urltodocid) { char cmd; int alloclen; char *urlbuf; if (urltodociddb == NULL) { cmd = C_DOCID_NODB; sendall(socket, &cmd, sizeof(cmd)); exit(1); } else { cmd = C_DOCID_READY; sendall(socket, &cmd, sizeof(cmd)); } cmd = C_DOCID_NEXT; alloclen = 1024; urlbuf = malloc(alloclen); do { unsigned int DocID; size_t len; if ((i = recv(socket, &cmd, sizeof(cmd), MSG_WAITALL)) == -1) { err(1, "recv(cmd)"); } if (cmd == C_DOCID_DONE) break; if ((i == recv(socket, &len, sizeof(len), MSG_WAITALL)) == -1) { err(1, "recv(len)"); } if (alloclen < len+1) { free(urlbuf); alloclen *= 2; urlbuf = malloc(alloclen); } if ((i == recv(socket, urlbuf, len, MSG_WAITALL)) == -1) { err(1, "recv(len)"); } urlbuf[len] = '\0'; if (!getDocIDFromUrl(urltodociddb, urlbuf, &DocID)) { cmd = C_DOCID_NOTFOUND; sendall(socket, &cmd, sizeof(cmd)); } else { cmd = C_DOCID_FOUND; sendall(socket, &cmd, sizeof(cmd)); sendall(socket, &DocID, sizeof(DocID)); } } while (1); free(urlbuf); } */ else { printf("unnown comand. %i\n", packedHedder.command); } //printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command); } //while }
int isOkCrawled (char subname[], struct gcaoptFormat *gcaopt) { char mysql_query [2048]; static MYSQL demo_db; MYSQL_RES *mysqlres; /* To be used to fetch information into */ MYSQL_ROW mysqlrow; int nrofcollections; int i,y; int crawler_success = 0; mysql_init(&demo_db); //koble til mysql if(!mysql_real_connect(&demo_db, MYSQL_HOST, MYSQL_USER, MYSQL_PASS, BOITHO_MYSQL_DB, 3306, NULL, 0)){ printf(mysql_error(&demo_db)); blog(gcaopt->logSummary,1,"MySQL Error: \"%s\".",mysql_error(&demo_db)); exit(1); } sprintf(mysql_query, "select crawler_success from shares where collection_name='%s'",subname); debug("mysql_query: %s\n",mysql_query); if(mysql_real_query(&demo_db, mysql_query, strlen(mysql_query))){ /* Make query */ printf(mysql_error(&demo_db)); blog(gcaopt->logSummary,1,"MySQL Error: \"%s\".",mysql_error(&demo_db)); exit(1); } mysqlres=mysql_store_result(&demo_db); /* Download result from server */ nrofcollections = (int)mysql_num_rows(mysqlres); if (nrofcollections == 0) { blog(gcaopt->logSummary,1,"Skipping \"%s\": dident find any rows",subname); } else if (nrofcollections != 1) { blog(gcaopt->logSummary,1,"Skipping \"%s\": has more then 1 rows",subname); } else { while ((mysqlrow=mysql_fetch_row(mysqlres)) != NULL) { /* Get a row from the results */ if (mysqlrow[0] != NULL) { crawler_success = atoi(mysqlrow[0]); } } } mysql_free_result(mysqlres); mysql_close(&demo_db); /* Fall back to fullyCrawled if mysql did not contain collection information */ if (nrofcollections == 0) { char path[PATH_MAX]; int fd; fprintf(stderr, "Falling back to using fullyCrawled file.\n"); GetFilPathForLot(path, 1, subname); strcat(path, "fullyCrawled"); fd = open(path, O_RDONLY); if (fd == -1) { crawler_success = 0; } else { crawler_success = 1; close(fd); } } printf("~isOkCrawled(crawler_success=%i)\n",crawler_success); if (crawler_success == 1) { return 1; } else { return 0; } }
void adultWeightopenMemArray(char servername[],char subname[]) { FILE *FH; int i; int LocalLots; char LotFile[128]; int branksize; off_t totsize = 0; struct stat inode; lotlistLoad(); lotlistMarkLocals(servername); //aloker minne for rank for de forskjelige lottene LocalLots=0; for (i=0;i<maxLots;i++) { //sjekekr om dette er en lokal lot if (lotlistIsLocal(i)) { GetFilPathForLot(LotFile,i,subname); strcat(LotFile,"AdultWeight"); // prøver å opne if ( (FH = fopen(LotFile,"r+b")) == NULL ) { perror(LotFile); adultWeightMemArray[i] = 0; } else { #ifdef DEBUG printf("loaded lot %i\n",i); #endif branksize = sizeof(unsigned char) * NrofDocIDsInLot; #ifdef MMAP_ADULT fstat(fileno(FH),&inode); if (inode.st_size < branksize) { fprintf(stderr,"adultWeightopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize); /* Stretch the file size to the size of the (mmapped) array of ints */ if (fseek(FH, branksize +1, SEEK_SET) == -1) { perror("Error calling fseek() to 'stretch' the file"); exit(EXIT_FAILURE); } /* Something needs to be written at the end of the file to * have the file actually have the new size. * Just writing an empty string at the current file position will do. * * Note: * - The current position in the file is at the end of the stretched * file due to the call to fseek(). * - An empty string is actually a single '\0' character, so a zero-byte * will be written at the last byte of the file. */ if (fwrite("", 1, 1, FH) != 1) { perror("Error writing last byte of the file"); exit(EXIT_FAILURE); } } if ((adultWeightMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == NULL) { fprintf(stderr,"adultWeightopenMemArray: can't mmap for lot %i\n",i); perror("mmap"); } #else if ((adultWeightMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) { printf("malloc eror for lot %i\n",i); perror("malloc"); exit(1); } fread(adultWeightMemArray[i],branksize,1,FH); #endif //debug: viser alle rankene vi laster //for(y=0;y<branksize;y++) { // printf("DocID %i, rank %i\n",y,adultWeightMemArray[i][y]); //} fclose(FH); totsize += branksize; ++LocalLots; } } else { adultWeightMemArray[i] = 0; } } printf("adultWeightopenMemArray: have %i local lots\n",LocalLots); printf("adultWeightopenMemArray: loaded a total of %"PRId64" bytes\n",totsize); }
main (int argc, char *argv[]) { char lotpath[512]; int lotNr; char file[512]; char *optHost = NULL; extern char *optarg; extern int optind, opterr, optopt; char c; while ((c=getopt(argc,argv,"h:"))!=-1) { switch (c) { case 'h': optHost = optarg; fprintf(stderr, "Will send to host %s\n",optHost); break; default: exit(1); } } --optind; if (argc -optind < 3) { printf("Dette programet kopierer en lot til en annen server\n\n\t./lotcp lotnr subname [file]\n\n"); printf("Options:\n\t-h host\n\n"); exit(0); } lotNr = atol(argv[1 +optind]); char *subname = argv[2 +optind]; //int rmkdir(char dest[], int LotNr,char subname[]); rmkdir("",lotNr,subname); //void GetFilPathForLot(char *FilePath,int LotNr,char subname[]); GetFilPathForLot(lotpath,lotNr,argv[2 +optind]); #ifdef DEBUG printf("lotpath %s\n",lotpath); #endif if ((argc -optind) == 3) { recursiveDir(lotpath,"",lotNr,subname,optHost); } else if ((argc -optind) == 4) { strcpy(file,argv[3 +optind]); //hvis vi er en mappe må vi slutte på /. Hvis vi ikke her det legger vi til en / if((strchr(file,'/') != NULL) && (file[strlen(file) -1] != '/')) { printf("adding a / to \"%s\"\n",file); strcat(file,"/"); } recursiveDir(lotpath,file,lotNr,subname,optHost); } else { fprintf(stderr,"wrong argc count.\n"); } }
//gir andre tilgan til lot filer. Casher opne filhandlere FILE *lotOpenFile(unsigned int DocID,char resource[],char type[], char lock,char subname[]) { int LotNr; int i; char FilePath[128]; char File [128]; if (!LotFilesInalisert) { for(i=0; i < MaxOpenFiles; i++) { OpenFiles[i].LotNr = -1; } LotFilesInalisert = 1; } File[0] = '\0'; //finner i hvilken lot vi skal lese fra LotNr = rLotForDOCid(DocID); //printf("LotNr: %i, DocID: %i\n",LotNr,DocID); //begynner med å søke cashen. Lopper til vi enten er ferdig, eller til vi har funne ønskede i cashen i = 0; while ((i < MaxOpenFiles) && (OpenFiles[i].LotNr != LotNr)) { i++; } //temp: skrur av søking her med i=0 //type of og subname er også lagt til uten at det tar hensyn til det i søket i = 0; //hvis vi fant i casehn returnerer vi den if (OpenFiles[i].LotNr == LotNr && (strcmp(OpenFiles[i].subname,subname) == 0) && (strcmp(OpenFiles[i].type,type)==0) && (strcmp(OpenFiles[i].resource,resource)==0) ) { #ifdef DEBUG printf("lotOpenFile: fant en tildigere åpnet fil, returnerer den.\n"); printf("lotOpenFile: returnerer: i %i, subname \"%s\", type \"%s\", LotNr %i\n",i,OpenFiles[i].subname,OpenFiles[i].type,OpenFiles[i].LotNr); printf("lotOpenFile: file is \"%s\"\n",OpenFiles[i].filename); printf("lotOpenFile: returning file handler %p\n",OpenFiles[i].FILEHANDLER); #endif if (OpenFiles[i].FILEHANDLER == NULL) { printf("Error: FILEHANDLER is NULL\n"); #ifdef DEBUG exit(-1); #endif } return OpenFiles[i].FILEHANDLER; } //hvis ikke åpner vi og returnerer else { //hvis dette er en åpen filhånterer, må vi lukke den if (OpenFiles[i].LotNr != -1) { printf("lotOpenFile: closeing: i %i\n",i); fclose(OpenFiles[i].FILEHANDLER); OpenFiles[i].LotNr = -1; } if ((OpenFiles[i].FILEHANDLER = lotOpenFileNoCasheByLotNr( LotNr, resource,type, lock,subname)) == NULL) { printf("lotOpenFileNoCashe: can't open file\n"); return NULL; } GetFilPathForLot(FilePath,LotNr,subname); strscpy(File,FilePath,sizeof(File)); strlcat(File,resource,sizeof(File)); strscpy(OpenFiles[i].filename,File,sizeof(OpenFiles[i].filename)); strscpy(OpenFiles[i].resource,resource,sizeof(OpenFiles[i].resource)); strscpy(OpenFiles[i].subname,subname,sizeof(OpenFiles[i].subname)); strscpy(OpenFiles[i].type,type,sizeof(OpenFiles[i].type)); //#ifdef DEBUG printf("lotOpenFile: opening file \"%s\" for %s\n",File,type); //#endif OpenFiles[i].LotNr = LotNr; return OpenFiles[i].FILEHANDLER; } }
int lotOpenFileNoCasheByLotNrl(int LotNr,char resource[],char type[], char lock,char subname[]) { int fd; int i; char FilePath[PATH_MAX]; //var 128 char File [PATH_MAX]; //var 128 #ifdef DEBUG printf("lotOpenFileNoCasheByLotNrl(LotNr=%i, subname=\"%s\", resource=\"%s\")\n",LotNr,subname,resource); #endif GetFilPathForLot(FilePath,LotNr,subname); strcpy(File,FilePath); strncat(File,resource,PATH_MAX); //var 128 #ifdef DEBUG printf("lotOpenFileNoCasheByLotNr: opening file \"%s\" for %s\n",File,type); #endif if (strcmp(type,">>") == 0) { //emulating perl's >>. If the file eksist is is opene for reading and writing. //if not it is createt and openf for writing and reading if ( (fd = open64(File,O_CREAT|O_RDWR,0664)) == -1 ) { makePath(FilePath); if ( (fd = open64(File,O_CREAT|O_RDWR,0664)) == -1 ) { perror(File); return -1; } } } //hvis dette er lesing så hjelper det ikke og prøve å opprette path. Filen vil fortsatt ikke finnes else if ((strcmp(type,"rb") == 0) || (strcmp(type,"r") == 0)) { if ( (fd = open64(File,O_RDONLY)) == -1 ) { #ifdef DEBUG perror(File); #endif return -1; } } //hvis dette er lesing så hjelper det ikke og prøve å opprette path. Filen vil fortsatt ikke finnes else if ((strcmp(type,"r+b") == 0) || (strcmp(type,"r+") == 0)) { if ( (fd = open64(File,O_RDWR)) == -1 ) { #ifdef DEBUG perror(File); #endif return -1; } } else { fprintf(stderr,"lotOpenFileNoCasheByLotNrl: ikke implementert\n"); exit(1); /* //temp: Bytte ut FilePath med filnavnet if ( (fd = open64(File,type)) == NULL ) { makePath(FilePath); //hvorfår har vi type "File" her ???, det verste er at det ser ut til å fungere også //if ( (FILEHANDLER = (FILE *)fopen64(File,"File")) == NULL ) { if ( (fd = (open64(File,type)) == NULL ) { perror(File); //exit(0); return NULL; } } */ } #ifdef DEBUG printf("lotOpenFile: tryint to obtain lock \"%c\"\n",lock); #endif //honterer låsning if (lock == 'e') { //skal vi ha flock64() her ? flock(fd,LOCK_EX); } else if (lock == 's') { flock(fd,LOCK_SH); } #ifdef DEBUG printf("lotOpenFile: lock obtained\n"); #endif #ifdef DEBUG printf("lotOpenFileNoCasheByLotNr: finished\n"); #endif return fd; }
void adultWeightopenMemArray2(char subname[]) { FILE *FH; int i, y ,z; int LocalLots; char LotFile[128]; int branksize; off_t totsize = 0; struct stat inode; int locklimit; //finner grensen på antal sider vi kan låse i minne struct rlimit rlim; if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) { printf("Warning: Cannot raise RLIMIT_MEMLOCK limits."); locklimit = 0; } else { locklimit = rlim.rlim_cur; } //aloker minne for rank for de forskjelige lottene LocalLots=0; for (i=0;i<maxLots;i++) { //sjekekr om dette er en lokal lot adultWeightMemArray[i] = 0; GetFilPathForLot(LotFile,i,subname); strcat(LotFile,"AdultWeight"); // prøver å opne if ( (FH = fopen(LotFile,"r+b")) == NULL ) { //perror(LotFile); continue; } fstat(fileno(FH),&inode); if (inode.st_size == 0) { printf("adultWeightopenMemArray2: file is emty\n"); continue; } //#ifdef DEBUG printf("loaded lot %i\n",i); //#endif branksize = sizeof(unsigned char) * NrofDocIDsInLot; #ifdef MMAP_ADULT if (inode.st_size < branksize) { fprintf(stderr,"adultWeightopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize); /* Stretch the file size to the size of the (mmapped) array of ints */ if (fseek(FH, branksize +1, SEEK_SET) == -1) { perror("Error calling fseek() to 'stretch' the file"); exit(EXIT_FAILURE); } /* Something needs to be written at the end of the file to * have the file actually have the new size. * Just writing an empty string at the current file position will do. * * Note: * - The current position in the file is at the end of the stretched * file due to the call to fseek(). * - An empty string is actually a single '\0' character, so a zero-byte * will be written at the last byte of the file. */ if (fwrite("", 1, 1, FH) != 1) { perror("Error writing last byte of the file"); exit(EXIT_FAILURE); } } if ((adultWeightMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == NULL) { fprintf(stderr,"adultWeightopenMemArray: can't mmap for lot %i\n",i); perror("mmap"); } //hvis vi kan låse uendelig med minne gjør vi det if (locklimit == -1) { //laster all rankerings dataen fra minne, slik ad det går fort å leste den inn siden z = 0; for(y=0;y<NrofDocIDsInLot;y++) { z += adultWeightMemArray[i][y]; } //låser minne if (mlock(adultWeightMemArray[i],branksize) != 0) { perror("mlock"); //exit(1); } } #else if ((adultWeightMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) { printf("malloc eror for lot %i\n",i); perror("malloc"); exit(1); } fread(adultWeightMemArray[i],branksize,1,FH); #endif //debug: viser alle rankene vi laster //for(y=0;y<branksize;y++) { // printf("DocID %i, rank %i\n",y,adultWeightMemArray[i][y]); //} fclose(FH); totsize += branksize; ++LocalLots; } printf("adultWeightopenMemArray: have %i local lots\n",LocalLots); printf("adultWeightopenMemArray: loaded a total of %"PRId64" bytes\n",totsize); }
int main (int argc, char *argv[]) { int lotNr; int lotPart; char path[256]; char iipath[256]; unsigned lastIndexTime; int optMustBeNewerThen = 0; int optAllowDuplicates = 0; struct revIndexArrayFomat *revIndexArray; revIndexArray = malloc(sizeof(struct revIndexArrayFomat) * revIndexArraySize); extern char *optarg; extern int optind, opterr, optopt; char c; while ((c=getopt(argc,argv,"nd"))!=-1) { switch (c) { case 'n': optMustBeNewerThen = 1; break; case 'd': optAllowDuplicates = 1; break; case 'v': break; default: exit(1); } } --optind; printf("lot %s, %i\n",argv[1],argc); char *type = argv[1 +optind]; lotNr = atoi(argv[2 +optind]); char *subname = argv[3 +optind]; if ((argc -optind)== 4) { //finner siste indekseringstid lastIndexTime = GetLastIndexTimeForLot(lotNr,subname); if(lastIndexTime == 0) { printf("lastIndexTime is 0\n"); exit(1); } //sjekker om vi har nokk palss if (!lotHasSufficientSpace(lotNr,4096,subname)) { printf("insufficient disk space\n"); exit(1); } printf("Indexing all buvkets for lot %i\n",lotNr); for (lotPart=0;lotPart<64;lotPart++) { //printf("indexint part %i for lot %i\n",lotPart,lotNr); //"$revindexPath/$revindexFilNr.txt"; GetFilPathForLot(path,lotNr,subname); //ToDo: må sette språk annen plass sprintf(iipath,"%siindex/%s/index/aa/",path,argv[1 +optind]); //oppretter paths makePath(iipath); sprintf(iipath,"%s%i.txt",iipath,lotPart); if ((optMustBeNewerThen != 0)) { if (fopen(iipath,"r") != NULL) { printf("we all redy have a iindex.\n"); continue; } } Indekser(iipath,revIndexArray,lotNr,type,lotPart,subname,optAllowDuplicates); } } else if ((argc - optind) == 5) { lotPart = atoi(argv[4 +optind]); printf("indexint part %i for lot %i\n",lotPart,lotNr); //"$revindexPath/$revindexFilNr.txt"; GetFilPathForLot(path,lotNr,subname); //ToDo: må sette språk annen plass //aa sprintf(iipath,"%siindex/%s/index/aa/%i.txt",path,argv[1 +optind],lotPart); //ToDo: må sette språk annen plass sprintf(iipath,"%siindex/%s/index/aa/",path,argv[1 +optind]); //oppretter paths makePath(iipath); sprintf(iipath,"%s%i.txt",iipath,lotPart); printf("iipath: \"%s\n",iipath); if ((optMustBeNewerThen != 0)) { if (fopen(iipath,"r") != NULL) { printf("we all redy have a iindex.\n"); exit(1); } } Indekser(iipath,revIndexArray,lotNr,type,lotPart,subname,optAllowDuplicates); } else { printf("usage: ./LotInvertetIndexMaker type lotnr subname [ lotPart ]\n\n"); } //GetFilPathForLot(lotNr); }
int gcrepo(int LotNr, char *subname) { int i; struct ReposetoryHeaderFormat ReposetoryHeader; char htmlbuffer[524288]; char imagebuffer[524288]; char *acl_allow; char *acl_deny; char *url, *attributes; unsigned long int raddress; char path[1024]; char path2[1024]; char path3[1024]; FILE *FNREPO; struct reformat *re; int keept = 0; int gced = 0; container *attrkeys = ropen(); if((re = reopen(LotNr, sizeof(struct DocumentIndexFormat), "DocumentIndex", subname, RE_HAVE_4_BYTES_VERSION_PREFIX|RE_COPYONCLOSE)) == NULL) { perror("reopen DocumentIndex"); return 0; } if ( (FNREPO = lotOpenFileNoCasheByLotNr(LotNr,"reposetory","rb", 's',subname)) == NULL) { #ifdef DEBUG printf("lot dont have a reposetory file\n"); #endif return 0; } while (rGetNext_fh(LotNr,&ReposetoryHeader,htmlbuffer,sizeof(htmlbuffer),imagebuffer,&raddress,0,0,subname,&acl_allow,&acl_deny, FNREPO ,&url, &attributes)) { #ifdef DEBUG printf("dokument \"%s\", DocID %u.\n", RE_DocumentIndex(re,ReposetoryHeader.DocID)->Url, ReposetoryHeader.DocID); #endif //printf("%p\n", docindex.RepositoryPointer); if (raddress != RE_DocumentIndex(re,ReposetoryHeader.DocID)->RepositoryPointer) { #ifdef DEBUG printf("Garbage collecting %d at %u. docindex has %u\n", ReposetoryHeader.DocID, raddress,RE_DocumentIndex(re,ReposetoryHeader.DocID)->RepositoryPointer); #endif ++gced; } else { unsigned long int offset; offset = rApendPost(&ReposetoryHeader, htmlbuffer, imagebuffer, subname, acl_allow, acl_deny, "repo.wip", url, attributes, attrkeys); RE_DocumentIndex(re,ReposetoryHeader.DocID)->RepositoryPointer = offset; #ifdef DEBUG printf("Writing DocID: %d\n", ReposetoryHeader.DocID); #endif ++keept; } } fclose(FNREPO); //lokker filen repo.wip //lotCloseFiles(); rclose(attrkeys); printf("keept %i\ngced %i\n",keept,gced); reclose(re); /* And we have a race... */ GetFilPathForLot(path, LotNr, subname); strcpy(path2, path); strcpy(path3, path); strcat(path, "repo.wip"); strcat(path2, "reposetory"); rename(path, path2); strcpy(path, path3); strcat(path, "DocumentIndex.wip"); strcat(path3, "DocumentIndex"); rename(path, path3); #ifdef DI_FILE_CASHE closeDICache(); #endif return 0; }
void popopenMemArray(char servername[], char subname[], char rankfile[]) { FILE *FH; int i; int LocalLots; char LotFile[128]; int branksize; struct stat inode; off_t totsize = 0; lotlistLoad(); lotlistMarkLocals(servername); //aloker minne for rank for de forskjelige lottene LocalLots=0; for (i=0;i<maxLots;i++) { //sjekekr om dette er en lokal lot if (lotlistIsLocal(i)) { GetFilPathForLot(LotFile,i,subname); strcat(LotFile,rankfile); // prøver å opne if ( (FH = fopen(LotFile,"rb")) == NULL ) { perror(LotFile); popMemArray[i] = 0; } else { #ifdef DEBUG printf("loaded lot %i\n",i); #endif branksize = sizeof(unsigned char) * NrofDocIDsInLot; #ifdef MMAP_POP fstat(fileno(FH),&inode); if (inode.st_size != branksize) { fprintf(stderr,"popopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize); continue; } if ((popMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == NULL) { fprintf(stderr,"popopenMemArray: can't mmap for lot %i\n",i); perror("popopenMemArray mmap"); } #else if ((popMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) { printf("malloc eror for lot %i\n",i); perror("malloc"); exit(1); } fread(popMemArray[i],branksize,1,FH); #endif //debug: viser alle rankene vi laster /* for(y=0;y<branksize;y++) { printf("DocID %i, rank %i\n",y,popMemArray[i][y]); } */ totsize += branksize; fclose(FH); ++LocalLots; } } else { popMemArray[i] = 0; } } printf("popopenMemArray: have %i local lots\n",LocalLots); printf("popopenMemArray: loaded a total of %"PRId64" bytes\n",totsize); }
void GetFilPathForLotFile(char *FilePath,char lotfile[],int LotNr,char subname[]) { GetFilPathForLot(FilePath,LotNr,subname); strcat(FilePath,lotfile); }
char *returnFilPathForLot(int LotNr,char subname[]) { static char FilePath[PATH_MAX]; GetFilPathForLot(FilePath,LotNr,subname); return FilePath; }
int main (int argc, char *argv[]) { int lotNr; int lotPart; char path[256]; char revpath[256]; char iipath[256]; unsigned lastIndexTime; struct revIndexArrayFomat *revIndexArray; revIndexArray = malloc(sizeof(struct revIndexArrayFomat) * revIndexArraySize); if (argc < 2) { } printf("lot %s, %i\n",argv[1],argc); if (argc == 3) { lotNr = atoi(argv[2]); //finner siste indekseringstid lastIndexTime = GetLastIndexTimeForLot(lotNr,subname); if(lastIndexTime == 0) { printf("lastIndexTime is 0\n"); exit(1); } //sjekker om vi har nokk palss if (!lotHasSufficientSpace(lotNr,4096,subname)) { printf("insufficient disk space\n"); exit(1); } printf("Indexing all buvkets for lot %i\n",lotNr); for (lotPart=0;lotPart<63;lotPart++) { //printf("indexint part %i for lot %i\n",lotPart,lotNr); //"$revindexPath/$revindexFilNr.txt"; GetFilPathForLot(path,lotNr,subname); sprintf(revpath,"%srevindex/%s/%i.txt",path,argv[1],lotPart); //ToDo: må sette språk annen plass sprintf(iipath,"%siindex/%s/index/aa/",path,argv[1]); //oppretter paths makePath(iipath); sprintf(iipath,"%s%i.txt",iipath,lotPart); Indekser(revpath,iipath,revIndexArray); //sletter revindex. Ingen vits i å ha den fylle opp plass //remove(revpath); } } else if (argc == 4) { lotNr = atoi(argv[2]); lotPart = atoi(argv[3]); printf("indexint part %i for lot %i\n",lotPart,lotNr); //"$revindexPath/$revindexFilNr.txt"; GetFilPathForLot(path,lotNr,subname); sprintf(revpath,"%srevindex/%s/%i.txt",path,argv[1],lotPart); //ToDo: må sette språk annen plass sprintf(iipath,"%siindex/%s/index/aa/%i.txt",path,argv[1],lotPart); Indekser(revpath,iipath,revIndexArray); } else { printf("usage: ./LotInvertetIndexMaker type lotnr [ lotPart ]\n\n"); } //GetFilPathForLot(lotNr); }
main (int argc, char *argv[]) { int LotNr; char lotPath[255]; char FileName[255]; struct ReposetoryHeaderFormat ReposetoryHeader; unsigned long int radress; char htmlbuffer[524288]; Bytef *htmlncompressBuffer; int htmlncompressBufferSize; char imagebuffer[524288]; int errornr; FILE *IMAGEFILE; FILE *HTMLFILE; if (argc < 3) { printf("Error ingen lotnr spesifisert.\n\nEksempel på bruk for å dumpe lot 2:\n\trread 2 /tmp/boitho/\n"); exit(1); } LotNr = atoi(argv[1]); GetFilPathForLot(&lotPath,LotNr); printf("Opning lot at: %s\n",lotPath); //henter minne htmlncompressBuffer = (Bytef *)malloc(2097152); //loppergjenom alle while (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,imagebuffer,&radress, 0, 0)) { printf("imageSize %i, htmlSize %i\n",ReposetoryHeader.imageSize,ReposetoryHeader.htmlSize); if (ReposetoryHeader.response == 200) { //skriver bilde sprintf(FileName,"%s%i.jpg",argv[2],ReposetoryHeader.DocID); if ((IMAGEFILE = fopen(FileName,"wb")) == NULL) { printf("Open file image file"); perror(FileName); exit(1); } fwrite(imagebuffer,ReposetoryHeader.imageSize,1,IMAGEFILE); fclose(IMAGEFILE); //skriver html fil sprintf(FileName,"%s%i.html",argv[2],ReposetoryHeader.DocID); if ((HTMLFILE = fopen(FileName,"wb")) == NULL) { printf("Open file image file"); perror(FileName); exit(1); } //dekomprimerer htmlen //burde ikke hardkode her htmlncompressBufferSize = 2097152; //printf("htmlncompressBufferSize: %i, htmlSize: %i\n",htmlncompressBufferSize, ReposetoryHeader.htmlSize); if ((errornr = uncompress(htmlncompressBuffer,(uLongf *)&htmlncompressBufferSize,htmlbuffer,(uLong)ReposetoryHeader.htmlSize)) != 0) { printf("Cant uncompress: %i\n",errornr); } else { fwrite(htmlncompressBuffer,htmlncompressBufferSize,1,HTMLFILE); fclose(HTMLFILE); } printf("DocId: %i url: %s\n",ReposetoryHeader.DocID,ReposetoryHeader.url); //sletter litt slik at vi ser om det blir noen feil i lesingen ReposetoryHeader.DocID = -1; } } free(htmlncompressBuffer); }