/*
Finner path for en lot fra docid
*/
void GetFilPathForLotByDocID(char *FilePath,int DocID,char subname[]) {
int lot;
lot = rLotForDOCid(DocID);
GetFilPathForLot(FilePath,lot,subname);
}
int bbdocument_deletecoll(char collection[]) {
int LotNr;
int i;
char FilePath[512];
char IndexPath[512];
char DictionaryPath[512];
FILE *fh;
debug("Deleting collection: \"%s\"\n",collection);
LotNr = 1;
while((fh =lotOpenFileNoCasheByLotNr(LotNr,"reposetory","r",'s',collection)) != NULL) {
GetFilPathForLot(FilePath,LotNr,collection);
fclose(fh);
rrmdir(FilePath);
++LotNr;
}
for (i=0; i < 64; i++) {
GetFilePathForIindex(FilePath,IndexPath,i,"Main","aa",collection);
#ifdef DEBUG
printf("FilePath: %s\nIndexPath: %s\n",FilePath,IndexPath);
#endif
if ((unlink(IndexPath) != 1) && (errno != ENOENT)) { //ENOENT=No such file or directory. Viser ikke feil hvis filen ikke fantes. Det er helt normalt
perror("remove IndexPath");
}
GetFilePathForIDictionary(FilePath,DictionaryPath,i,"Main","aa",collection);
#ifdef DEBUG
printf("FilePath: %s\nDictionaryPath: %s\n",FilePath,DictionaryPath);
#endif
if ((unlink(DictionaryPath) != 0) && (errno != ENOENT)) {//ENOENT=No such file or directory. Viser ikke feil hvis filen ikke fantes. Det er helt normalt
perror("remove DictionaryPath");
}
}
//sletter i userToSubname.db
struct userToSubnameDbFormat userToSubnameDb;
if (!userToSubname_open(&userToSubnameDb,'w')) {
printf("can't open users.db\n");
}
else {
userToSubname_deletecol(&userToSubnameDb,collection);
userToSubname_close(&userToSubnameDb);
}
return 1;
}
main (int argc, char *argv[]) {
int LotNr;
char lotPath[255];
struct ReposetoryHeaderFormat ReposetoryHeader;
unsigned int radress;
char htmlbuffer[524288];
char imagebuffer[524288];
char subname[maxSubnameLength];
struct udfileFormat udfilepost;
FILE *FH;
if (argc < 3) {
printf("Error ingen lotnr spesifisert.\n\nEksempel på bruk for å lese lot 2:\n\trread 2 www\n");
exit(1);
}
LotNr = atoi(argv[1]);
strncpy(subname,argv[2],sizeof(subname) -1);
printf("lotnr %i\n",LotNr);
GetFilPathForLot(lotPath,LotNr,subname);
printf("Opning lot at: %s for %s\n",lotPath,subname);
if ((FH = fopen(FilePath,"ab")) == NULL) {
perror("tmpe file");
exit(1);
}
//loppergjenom alle
int htmlbufferSize = sizeof(htmlbuffer);
int count = 0;
while (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,htmlbufferSize,imagebuffer,&radress,0,0,subname)) {
if ((ReposetoryHeader.response == 200) && (ReposetoryHeader.imageSize == 0) && (ReposetoryHeader.htmlSize != 0)) {
//printf("DocId: %i url: %s res %hi htmls %hi time %lu\n",ReposetoryHeader.DocID,ReposetoryHeader.url,ReposetoryHeader.response,ReposetoryHeader.htmlSize,ReposetoryHeader.time);
//printf("################################\n%s##############################\n",htmlbuffer);
strncpy(udfilepost.url,ReposetoryHeader.url,sizeof(udfilepost.url));
udfilepost.DocID = ReposetoryHeader.DocID;
fwrite(&udfilepost,sizeof(udfilepost),1,FH);
}
++count;
}
printf("Did analyse %i pages\n",count);
fclose(FH);
}
int
lotOpenFileNoCache_direct(unsigned int DocID, char *resource, char *type, char lock, char *subname)
{
unsigned int LotNr = rLotForDOCid(DocID);
int i;
char FilePath[PATH_MAX];
char File [PATH_MAX];
int fd;
printf("lotOpenFileNoCache_direct(subname: \"%s\", resource %s)\n",subname,resource);
GetFilPathForLot(FilePath,LotNr,subname);
strcpy(File,FilePath);
strncat(File,resource,PATH_MAX); //var 128
#ifdef DEBUG
printf("lotOpenFileNoCasheByLotNr: opening file \"%s\" for %s\n",File,type);
#endif
//hvis dette er lesing så hjelper det ikke og prøve å opprette path. Filen vil fortsatt ikke finnes
if ((strcmp(type,"rb") == 0) || (strcmp(type,"r") == 0)) {
if ((fd = open64(File, O_RDONLY|O_DIRECT|O_LARGEFILE)) == -1) {
warn("open64: %d", fd);
#ifdef DEBUG
perror(File);
#endif
return -1;
}
} else {
errx(1, "We can only open this for reading right now");
}
#ifdef DEBUG
printf("lotOpenFile: tryint to obtain lock \"%c\"\n",lock);
#endif
//honterer låsning
if (lock == 'e') {
//skal vi ha flock64() her ?
flock(fd, LOCK_EX);
}
else if (lock == 's') {
flock(fd, LOCK_SH);
}
#ifdef DEBUG
printf("lotOpenFile: lock obtained\n");
#endif
#ifdef DEBUG
printf("lotOpenFileNoCasheByLotNr: finished\n");
#endif
return fd;
}
void makeLotPath(int lotNr,char folder[],char subname[]) {
char path[PATH_MAX];
GetFilPathForLot(path,lotNr,subname);
strcat(path,"/");
strcat(path,folder);
makePath(path);
}
main (int argc, char *argv[]) {
int LotNr;
char lotPath[255];
int rank;
struct ReposetoryHeaderFormat ReposetoryHeader;
unsigned int radress;
char htmlbuffer[524288];
char imagebuffer[524288];
char *acl;
if (argc < 5) {
printf("Error ingen lotnr spesifisert.\n\nEksempel på bruk for å lese lot 2:\n\trread 2 www servername minrank\n");
exit(1);
}
LotNr = atoi(argv[1]);
char *subname = argv[2];
char *servername = argv[3];
int minrank = atoi(argv[4]);
printf("lotnr %i\n",LotNr);
GetFilPathForLot(lotPath,LotNr,subname);
printf("Opning lot at: %s for %s\n",lotPath,subname);
popopenMemArray(servername,"www");
//loppergjenom alle
// int rGetNext (unsigned int LotNr,struct ReposetoryHeaderFormat *ReposetoryHeader, char htmlbuffer[],
// int htmlbufferSize, char imagebuffer[], unsigned int *radress,
// unsigned int FilterTime, unsigned int FileOffset, char subname[]);
while (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,sizeof(htmlbuffer),imagebuffer,&radress,0,0,subname,&acl)) {
rank = popRankForDocIDMemArray(ReposetoryHeader.DocID);
if (rank >= minrank) {
//printf("DocId: %i url: %s res %hi htmls %hi time %lu, rank %i\n",ReposetoryHeader.DocID,ReposetoryHeader.url,ReposetoryHeader.response,ReposetoryHeader.htmlSize,ReposetoryHeader.time,rank);
printf("%s\n",ReposetoryHeader.url);
}
//printf("################################\n%s##############################\n",htmlbuffer);
}
}
//gir ful path for et bilde fra DocID
void GetFilPathForThumbnaleByDocID(char *FileName,int DocID,char subname[]) {
int LotNr;
int ImageBucket;
ImageBucket = fmod(DocID,512);
//finner path
LotNr = rLotForDOCid(DocID);
GetFilPathForLot(FileName,LotNr,subname);
sprintf(FileName,"%simages/%i/%i.jpg",FileName,ImageBucket,DocID);
}
void
cache_fresh_lot_collection(void)
{
DIR *colls;
char *coll;
char path[2048];
size_t len;
int i;
/* We only look at the first 5 lots */
if ((colls = listAllColl_start()) == NULL) {
bblog(ERROR, "Can't listAllColl_start()");
return;
}
while ((coll = listAllColl_next(colls))) {
for (i = 1; i < 6; i++) {
DIR *dirp;
struct dirent *de;
GetFilPathForLot(path, i, coll);
len = strlen(path);
dirp = opendir(path);
if (dirp == NULL)
continue;
while ((de = readdir(dirp))) {
int fd;
int dw;
if (de->d_name[0] == '.')
continue;
sprintf(path+len, "/%s", de->d_name);
bblog(DEBUGINFO, "Found file: %s", path);
fd = open(path, O_RDONLY);
if (fd == -1)
continue;
read(fd, &dw, sizeof(dw));
close(fd);
}
closedir(dirp);
}
}
listAllColl_close(colls);
}
int lotHasSufficientSpace(int lot, int needSpace,char subname[]) {
char FilePath[512];
int i;
GetFilPathForLot(FilePath,lot,subname);
//må lage, slik at vi er 100% sikker på at vi har noe å teste mot
#ifdef DEBUG
printf("will hav to make lot path sow we can test for space\n");
#endif
makePath(FilePath);
i=HasSufficientSpace(FilePath,needSpace);
return i;
}
//sjeker om det fins en DocumentIndex fro denne loten
int DIHaveIndex (int lotNr,char subname[]) {
char FilePath[512];
FILE *FH;
GetFilPathForLot(FilePath,lotNr,subname);
strncat(FilePath,"DocumentIndex",sizeof(FilePath));
if ((FH = fopen(FilePath,"r")) == NULL) {
return 0;
}
else {
fclose(FH);
return 1;
}
}
int
main(int argc, char **argv)
{
FILE *fp;
struct redirects redir;
char *subname;
int LotNr;
char filename[1024];
char *text;
size_t len;
if (argc < 2)
err(1, "Usage: ./readtest lotnr subname");
LotNr = atoi(argv[1]);
subname = argv[2];
GetFilPathForLot(filename, LotNr, subname);
strcat(filename, "redirmap");
if ((fp = fopen(filename, "r")) == NULL)
err(1, "fopen(%s)", filename);
len = 1048576 * 5; //5MB
if ((text = malloc(len)) == NULL) {
perror("malloc");
exit(1);
}
while (fread(&redir, sizeof(redir), 1, fp) == 1) {
int len2;
printf("%u => %u (reason: %hu)\n", redir.DocID, redir.redirectTo, redir.response);
len2 = readHTMLNET(subname, redir.redirectTo, text, len);
printf("fiii: %d\n", len2);
if (text[0] != '\0')
addResource(LotNr, subname, redir.DocID, text, len2);
// printf("And got html: ''%s''\n", text);
}
free(text);
fclose(fp);
return 0;
}
void popopenMemArray_oneLot(char subname[], int i) {
FILE *FH;
char LotFile[128];
int branksize;
GetFilPathForLot(LotFile,i,subname);
strcat(LotFile,"Brank");
// prøver å opne
if ( (FH = fopen(LotFile,"rb")) == NULL ) {
perror(LotFile);
popMemArray[i] = 0;
}
else {
#ifdef DEBUG
printf("loaded lot %i\n",i);
#endif
branksize = sizeof(unsigned char) * NrofDocIDsInLot;
if ((popMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) {
printf("malloc eror for lot %i\n",i);
perror("malloc");
exit(1);
}
fread(popMemArray[i],branksize,1,FH);
//debug: viser alle rankene vi laster
/*
int y;
for(y=0;y<NrofDocIDsInLot;y++) {
printf("DocID %i, rank %u. i:%i, y:%i\n",(y + LotDocIDOfset(i)),(unsigned int)popMemArray[i][y],i,y);
}
*/
fclose(FH);
}
}
void revindexFilesOpenLocal(FILE *revindexFilesHa[],int lotNr) {
int i;
char lotPath[128];
char revfile[128];
GetFilPathForLot(lotPath,lotNr);
//oppretter path
sprintf(revfile,"%srevindex/Main/",lotPath);
makePath(revfile);
for(i=0;i<NrOfDataDirectorys;i++) {
sprintf(revfile,"%srevindex/Main/%i.txt",lotPath,i);
//printf("opning revfile %s\n",revfile);
if ((revindexFilesHa[i] = fopen(revfile,"wb")) == NULL) {
perror(revfile);
exit(1);
}
}
}
int
main(int argc, char **argv) {
int LotNr;
char lotPath[255];
struct DocumentIndexFormat docindex;
unsigned int DocID;
char text[40];
unsigned int radress;
if (argc < 3)
errx(1, "Usage: ./DIread lotNr subname");
#if 0
if (argc > 3)
anchoraddnew(125500001, "eirik", 5, "www", NULL);
#endif
LotNr = atoi(argv[1]);
char *subname = argv[2];
printf("lotnr %i\n",LotNr);
GetFilPathForLot(lotPath,LotNr,subname);
printf("Opning lot at: %s for %s\n",lotPath,subname);
#if 1
//loppergjenom alle
while (DIGetNext(&docindex, LotNr, &DocID, subname)) {
printf("DocID: %d\n", DocID);
if (anchorRead(LotNr, subname, DocID, text, sizeof(text)))
printf("\tAnchor text: '%s'\n", text);
}
#endif
return 0;
}
void popopenMemArray2(char subname[], char rankfile[]) {
FILE *FH;
int i, y, z;
int LocalLots;
char LotFile[128];
int branksize;
struct stat inode;
off_t totsize = 0;
int locklimit;
//finner grensen på antal sider vi kan låse i minne
struct rlimit rlim;
if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
printf("Warning: Cannot raise RLIMIT_MEMLOCK limits.");
locklimit = 0;
}
else {
locklimit = rlim.rlim_cur;
}
//aloker minne for rank for de forskjelige lottene
LocalLots=0;
for (i=0;i<maxLots;i++) {
//sjekekr om dette er en lokal lot
popMemArray[i] = 0;
GetFilPathForLot(LotFile,i,subname);
strcat(LotFile,rankfile);
// prøver å opne
if ( (FH = fopen(LotFile,"rb")) == NULL ) {
#ifdef DEBUG
perror(LotFile);
#endif
continue;
}
fstat(fileno(FH),&inode);
if (inode.st_size == 0) {
printf("file is emty\n");
continue;
}
#ifdef DEBUG
printf("popopenMemArray2: loaded lot %i\n",i);
#endif
branksize = sizeof(unsigned char) * NrofDocIDsInLot;
printf("branksize offset: %i\n",branksize % getpagesize());
#ifdef MMAP_POP
//vi må aligne dette med pagesize
//branksize += branksize % getpagesize();
if (inode.st_size != branksize) {
fprintf(stderr,"popopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize);
continue;
}
//MAP_LOCKED (since Linux 2.5.37)
// Lock the pages of the mapped region into memory in the manner of mlock(). This flag is ignored in older kernels.
if ((popMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == MAP_FAILED) {
fprintf(stderr,"popopenMemArray: can't mmap for lot %i\n",i);
perror("popopenMemArray2 mmap");
}
//hvis vi kan låse uendelig med minne gjør vi det
if (locklimit == -1) {
//laster all rankerings dataen fra minne, slik ad det går fort å leste den inn siden
z = 0;
for(y=0;y<NrofDocIDsInLot;y++) {
z += popMemArray[i][y];
}
//låser minne
if (mlock(popMemArray[i],branksize) != 0) {
perror("mlock");
}
}
#else
if ((popMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) {
printf("malloc eror for lot %i\n",i);
perror("malloc");
exit(1);
}
fread(popMemArray[i],branksize,1,FH);
#endif
//debug: viser alle rankene vi laster
/*
for(y=0;y<branksize;y++) {
printf("DocID %i, rank %i\n",y,popMemArray[i][y]);
}
*/
totsize += branksize;
fclose(FH);
++LocalLots;
}
printf("popopenMemArray: have %i local lots\n",LocalLots);
printf("popopenMemArray: loaded a total of %"PRId64" bytes\n",totsize);
}
void connectHandler(int socket) {
struct packedHedderFormat packedHedder;
int isAuthenticated = 0;
char tkeyForTest[32];
int i,n;
int intrespons;
int count = 0;
container *attrkeys = NULL;
#ifdef DEBUG_TIME
struct timeval start_time, end_time;
struct timeval tot_start_time, tot_end_time;
gettimeofday(&tot_start_time, NULL);
#endif
ionice_benice();
while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) {
#ifdef DEBUG
printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
#endif
packedHedder.size = packedHedder.size - sizeof(packedHedder);
if (attrkeys == NULL) {
attrkeys = ropen();
}
if (packedHedder.command == bbc_askToAuthenticate) {
if ((i=recv(socket, tkeyForTest, sizeof(tkeyForTest),MSG_WAITALL)) == -1) {
perror("Cant read tkeyForTest");
exit(1);
}
if (1) {
printf("authenticated\n");
intrespons = bbc_authenticate_ok;
bbdocument_init(NULL);
isAuthenticated = 1;
}
else {
printf("authenticate faild\n");
intrespons = bbc_authenticate_feiled;
}
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else {
if (!isAuthenticated) {
printf("user not autentikated\n");
exit(1);
}
if (packedHedder.command == bbc_docadd) {
#ifdef DEBUG
printf("bbc_docadd\n");
#endif
char *subname,*documenturi,*documenttype,*document,*acl_allow,*acl_denied,*title,*doctype;
char *attributes;
int dokument_size;
unsigned int lastmodified;
#ifdef DEBUG_TIME
gettimeofday(&start_time, NULL);
#endif
//subname
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recvall(socket, subname, intrespons)) == 0) {
perror("Cant read subname");
exit(1);
}
//documenturi
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
documenturi = malloc(intrespons +1);
if ((i=recvall(socket, documenturi, intrespons)) == 0) {
perror("Cant read documenturi");
exit(1);
}
//documenttype
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
documenttype = malloc(intrespons +1);
if ((i=recvall(socket, documenttype, intrespons)) == 0) {
perror("Cant read documenttype");
exit(1);
}
//document
//dokument_size
if ((i=recvall(socket, &dokument_size, sizeof(dokument_size))) == 0) {
perror("Cant read dokument_size");
exit(1);
}
document = malloc(dokument_size +1);
if (dokument_size == 0) {
document[0] = '\0';
}
else {
if ((i=recvall(socket, document, dokument_size)) == 0) {
fprintf(stderr,"Can't read document of size %i\n",dokument_size);
perror("recvall");
exit(1);
}
}
//lastmodified
if ((i=recvall(socket, &lastmodified, sizeof(lastmodified))) == 0) {
perror("Cant read lastmodified");
exit(1);
}
//acl_allow
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
acl_allow = malloc(intrespons +1);
if ((i=recvall(socket, acl_allow, intrespons)) == 0) {
perror("Cant read acl_allow");
exit(1);
}
//acl_denied
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
acl_denied = malloc(intrespons +1);
if ((i=recvall(socket, acl_denied, intrespons)) == 0) {
perror("Cant read acl_denied");
exit(1);
}
//title
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
title = malloc(intrespons +1);
if ((i=recvall(socket, title, intrespons)) == 0) {
perror("Cant read title");
exit(1);
}
//doctype
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
doctype = malloc(intrespons +1);
if ((i=recvall(socket, doctype, intrespons)) == 0) {
perror("Cant read doctype");
exit(1);
}
// Attribute list
if ((i = recvall(socket, &intrespons, sizeof(intrespons))) == 0)
err(1, "Can't receive attribute list len");
attributes = malloc(intrespons +1);
if ((i=recvall(socket, attributes, intrespons)) == 0)
err(1, "Can't receive attribute list");
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: bbdn_docadd recv data time: %f\n",getTimeDifference(&start_time, &end_time));
#endif
printf("\n");
printf("########################################################\n");
printf("Url: %s\n",documenturi);
printf("got subname \"%s\": title \"%s\". Nr %i, dokument_size %i attrib: %s\n",subname,title,count,dokument_size, attributes);
printf("########################################################\n");
printf("calling bbdocument_add():\n");
#ifdef DEBUG_TIME
gettimeofday(&start_time, NULL);
#endif
intrespons = bbdocument_add(subname,documenturi,documenttype,document,dokument_size,lastmodified,acl_allow,acl_denied,title,doctype, attributes, attrkeys);
printf(":bbdocument_add end\n");
printf("########################################################\n");
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: bbdn_docadd runing bbdocument_add() time: %f\n",getTimeDifference(&start_time, &end_time));
#endif
free(subname);
free(documenturi);
free(documenttype);
free(document);
free(acl_allow);
free(acl_denied);
free(title);
free(doctype);
free(attributes);
// send status
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_opencollection) {
char *subname;
char path[PATH_MAX];
printf("open collection\n");
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1)
err(1, "Cant read intrespons");
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1)
err(1, "Cant read subname");
GetFilPathForLot(path, 1, subname);
strcat(path, "fullyCrawled");
unlink(path);
free(subname);
}
else if (packedHedder.command == bbc_closecollection) {
printf("closecollection\n");
char *subname;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
bbdocument_close(attrkeys);
attrkeys = NULL;
//toDo må bruke subname, og C ikke perl her
printf("cleanin lots start\n");
char command[PATH_MAX];
snprintf(command,sizeof(command),"perl %s -l -s \"%s\"",bfile("perl/cleanLots.pl"),subname);
printf("running \"%s\"\n",command);
intrespons = system(command);
printf("cleanin lots end\n");
// legger subnamet til listen over ventene subnavn, og huper searchd.
lot_recache_collection(subname);
/* We are done crawling */
{
int fd = lotOpenFileNoCasheByLotNrl(1, "fullyCrawled", ">>", '\0', subname);
if (fd == -1) {
warn("Unable to write fullyCrawled file");
} else {
close(fd);
}
}
free(subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_deleteuri) {
printf("deleteuri\n");
char *subname, *uri;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
uri = malloc(intrespons +1);
if ((i=recv(socket, uri, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read uri");
exit(1);
}
uri[intrespons] = '\0';
printf("going to delete: %s from %s\n", uri, subname);
/* Add docid to the gced file */
{
FILE *fh;
unsigned int DocID, lastmodified;
unsigned int lotNr;
int err = 0;
if (uriindex_get(uri, &DocID, &lastmodified, subname) == 0) {
fprintf(stderr,"Unable to get uri info. uri=\"%s\",subname=\"%s\".",uri,subname);
perror("Unable to get uri info");
err++;
}
if (!err) {
lotNr = rLotForDOCid(DocID);
if ((fh = lotOpenFileNoCasheByLotNr(lotNr,"gced","a", 'e',subname)) == NULL) {
perror("can't open gced file");
err++;
} else {
fwrite(&DocID, sizeof(DocID), 1, fh);
fclose(fh);
}
}
if (!err) {
struct reformat *re;
if((re = reopen(rLotForDOCid(DocID), sizeof(struct DocumentIndexFormat), "DocumentIndex", subname, RE_HAVE_4_BYTES_VERSION_PREFIX)) == NULL) {
perror("can't reopen()");
err++;
} else {
DIS_delete(RE_DocumentIndex(re, DocID));
reclose(re);
}
}
//markerer at den er skitten
if (!err) {
FILE *dirtfh;
dirtfh = lotOpenFileNoCashe(DocID,"dirty","ab",'e',subname);
fwrite("1",1,1,dirtfh);
fclose(dirtfh);
}
if (err == 0)
bbdocument_delete(uri, subname);
}
free(subname);
intrespons = 1; // Always return ok for now
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_deletecollection) {
printf("deletecollection\n");
char *subname, *uri;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
printf("going to delete collection: %s\n", subname);
intrespons = bbdocument_deletecoll(subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
free(subname);
}
else if (packedHedder.command == bbc_addwhisper) {
whisper_t add;
char *subname;
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1)
err(1, "Cant read intrespons");
subname = malloc(intrespons+1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
if ((i=recv(socket, &add, sizeof(add),MSG_WAITALL)) == -1)
err(1, "Cant read add whisper");
gcwhisper_write(subname, add);
free(subname);
}
else if (packedHedder.command == bbc_HasSufficientSpace) {
char *subname;
//subname
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recvall(socket, subname, intrespons)) == 0) {
perror("Cant read subname");
exit(1);
}
// tester bare i lot 1 her. Må også sjekke andre loter når vi begynner å støtte frlere disker på ES.
intrespons = lotHasSufficientSpace(1, 4096, subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
printf("~Asked for HasSufficientSpace for subname \"%s\". Returnerer %d\n",subname, intrespons);
free(subname);
}
else {
printf("unnown comand. %i\n", packedHedder.command);
}
}
++count;
// #ifdef DEBUG_BREAK_AFTER
// if (count >= DEBUG_BREAK_AFTER) {
// printf("exeting after %i docoments\n",count);
// exit(1);
// }
// #endif
}
#ifdef DEBUG_TIME
gettimeofday(&tot_end_time, NULL);
printf("Time debug: bbdn total time time: %f\n",getTimeDifference(&tot_start_time, &tot_end_time));
#endif
}
void connectHandler(int socket) {
struct packedHedderFormat packedHedder;
int i,n;
int LotNr;
char lotPath[512];
char buf[100];
unsigned int FilterTime;
int filnamelen;
FILE *FH;
struct stat inode; // lager en struktur for fstat å returnere.
off_t filesize;
char c;
struct DocumentIndexFormat DocumentIndexPost;
int DocID;
struct ReposetoryHeaderFormat ReposetoryHeader;
unsigned int radress;
char htmlbuffer[524288];
int destLeng;
char dest[512];
off_t fileBloks,filerest;
char *filblocbuff;
//while ((i=read(socket, &packedHedder, sizeof(struct packedHedderFormat))) > 0) {
while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) {
//printf("command: %i\n",packedHedder.command);
//printf("i er %i\n",i);
printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
//printf("subname: %s\n",packedHedder.subname);
//lar size reflektere hva som er igjen av pakken
packedHedder.size = packedHedder.size - sizeof(packedHedder);
if (packedHedder.command == C_rmkdir) {
printf("C_rmkdir\n");
//leser data. Det skal væren en int som sier hvilken lot vi vil ha
if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) {
perror("Cant read lotnr");
exit(1);
}
//leser destinasjonelengden
if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) {
perror("Cant read destLeng");
exit(1);
}
if (destLeng > sizeof(dest)) {
printf("dest filname is to long at %i\n",destLeng);
exit(1);
}
//leser destinasjonene
if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) {
perror("Cant read dest");
exit(1);
}
GetFilPathForLot(lotPath,LotNr,packedHedder.subname);
sprintf(lotPath,"%s%s",lotPath,dest);
printf("mkdir %s\n",lotPath);
makePath(lotPath);
printf("~C_rmkdir\n");
}
else if (packedHedder.command == C_rComand) {
//leser data. Det skal væren en int som sier hvilken lot vi vil ha
if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) {
perror("Cant read lotnr");
exit(1);
}
//leser destinasjonelengden
if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) {
perror("Cant read destLeng");
exit(1);
}
if (destLeng > sizeof(dest)) {
printf("dest filname is to long at %i\n",destLeng);
exit(1);
}
//leser destinasjonene
if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) {
perror("Cant read dest");
exit(1);
}
printf("run command %s\n",dest);
system(dest);
}
else if (packedHedder.command == C_getLotToIndex) {
printf("fikk C_getLotToIndex\n");
int dirty;
if ((i=recv(socket, &dirty, sizeof(dirty),MSG_WAITALL)) == -1) {
perror("Cant read dirty");
exit(1);
}
printf("dirty: %i\n",dirty);
LotNr = findLotToIndex(packedHedder.subname,dirty);
printf("sending respons\n");
sendall(socket,&LotNr, sizeof(LotNr));
}
else if (packedHedder.command == C_getlotHasSufficientSpace) {
printf("fikk C_getLotToIndex\n");
int needSpace;
int response;
if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
perror("Cant read lotnr");
exit(1);
}
if ((i=recv(socket, &needSpace, sizeof(needSpace),MSG_WAITALL)) == -1) {
perror("Cant read dirty");
exit(1);
}
printf("needSpace: %i, LotNr %i\n",needSpace,LotNr);
response = lotHasSufficientSpace(LotNr, needSpace, packedHedder.subname);
printf("sending respons\n");
sendall(socket,&response, sizeof(response));
}
else if (packedHedder.command == C_rGetSize) {
printf("fikk C_rGetSize\n");
//leser data. Det skal væren en int som sier hvilken lot vi vil ha
if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
perror("Cant read lotnr");
exit(1);
}
if ((i=read(socket, &filnamelen, sizeof(filnamelen))) == -1) {
perror("Cant read filnamelen");
exit(1);
}
if (filnamelen > sizeof(buf)) {
printf("filname to long\n");
};
if ((i=read(socket, buf, filnamelen)) == -1) {
perror("Cant read filnamelen");
exit(1);
}
printf("filname %s\n",buf);
if ((FH = lotOpenFileNoCasheByLotNr(LotNr,buf,"rb",'s',packedHedder.subname)) == NULL) {
perror(buf);
//sending that he fil is emty
fileBloks = 0;
sendall(socket,&fileBloks, sizeof(fileBloks));
}
else {
//finner og sender il størelse
fstat(fileno(FH),&inode);
//filesize = inode.st_size;
//sendall(socket,&filesize, sizeof(filesize));
fileBloks = inode.st_size;
printf("size is %" PRId64 "\n",fileBloks);
sendall(socket,&fileBloks, sizeof(fileBloks));
fclose(FH);
}
}
else if (packedHedder.command == C_rGetFile) {
printf("fikk C_rGetFile\n");
//leser data. Det skal væren en int som sier hvilken lot vi vil ha
if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
perror("Cant read lotnr");
exit(1);
}
if ((i=read(socket, &filnamelen, sizeof(filnamelen))) == -1) {
perror("Cant read filnamelen");
exit(1);
}
if (filnamelen > sizeof(buf)) {
printf("filname to long\n");
};
if ((i=read(socket, buf, filnamelen)) == -1) {
perror("Cant read filnamelen");
exit(1);
}
printf("filname %s\n",buf);
if ((FH = lotOpenFileNoCasheByLotNr(LotNr,buf,"rb",'s',packedHedder.subname)) == NULL) {
perror(buf);
//sending that the fil is emty
fileBloks = 0;
filerest = 0;
sendall(socket,&fileBloks, sizeof(fileBloks));
sendall(socket,&filerest, sizeof(filerest));
}
else {
//finner og sender fil størelse
fstat(fileno(FH),&inode);
//filesize = inode.st_size;
//sendall(socket,&filesize, sizeof(filesize));
fileBloks = (int)floor(inode.st_size / rNetTrabsferBlok);
filerest = inode.st_size - (fileBloks * rNetTrabsferBlok);
sendall(socket,&fileBloks, sizeof(fileBloks));
sendall(socket,&filerest, sizeof(filerest));
printf("sending fil. fileBloks %"PRId64", filerest %"PRId64"\n",fileBloks,filerest);
filblocbuff = (char *)malloc(rNetTrabsferBlok);
for(i=0; i < fileBloks; i++) {
//fread(filblocbuff,sizeof(c),rNetTrabsferBlok,FH);
//fread_all(const void *buf, size_t size, FILE *stream)
fread_all(filblocbuff,rNetTrabsferBlok,FH, 4096);
if ((n=sendall(socket, filblocbuff, rNetTrabsferBlok)) == -1) {
perror("Cant recv dest");
exit(1);
}
}
printf("did recv %i fileBloks\n",i);
fread(filblocbuff,sizeof(c),filerest,FH);
if ((n=sendall(socket, filblocbuff, filerest)) == -1) {
perror("Cant recv filerest");
exit(1);
}
free(filblocbuff);
/*
for (i=0;i<filesize;i++) {
fread(&c,sizeof(char),1,FH);
send(socket, &c, sizeof(char), 0);
//printf("%i\n",(int)c);
}
*/
printf("send file end\n");
fclose(FH);
}
}
else if (packedHedder.command == C_rGetNext) {
printf("fikk C_rGetNext\n");
printf("støttes ikke lengere");
exit(1);
/*
//leser data. Det skal væren en unigned int som sier hvilken lot vi vil ha
//har deklarert den som int her ???
if ((i=read(socket, &LotNr, sizeof(LotNr))) == -1) {
perror("Cant read lotnr");
exit(1);
}
printf("leser FilterTime\n");
//leser filtertime
if ((i=read(socket, &FilterTime, sizeof(FilterTime))) == -1) {
perror("Cant read lotnr");
exit(1);
}
printf("lotnr %i FilterTime %u\n",LotNr,FilterTime);
//henter inn data om den lotten
if (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,NULL,&radress,FilterTime,0)) {
//printf("DocId: %i url: %s\n",ReposetoryHeader.DocID,ReposetoryHeader.url);
//sender pakke hedder
sendpacked(socket,C_rLotData,PROTOCOLVERSION, ReposetoryHeader.htmlSize + sizeof(ReposetoryHeader) +sizeof(radress), NULL,packedHedder.subname);
//sennder ReposetoryHeader'en
sendall(socket,&ReposetoryHeader, sizeof(ReposetoryHeader));
//sender htmlen
sendall(socket,&htmlbuffer, ReposetoryHeader.htmlSize);
//sender adressen
sendall(socket,&radress,sizeof(radress));
//printf("data sent\n");
//printf("rGetNext: %i\n",ReposetoryHeader.DocID);
}
else {
sendpacked(socket,C_rEOF,PROTOCOLVERSION, 0, NULL,packedHedder.subname);
printf("ferdig\n");
}
*/
}
else if (packedHedder.command == C_DIWrite) {
if ((i=recv(socket, &DocumentIndexPost, sizeof(struct DocumentIndexFormat),MSG_WAITALL)) == -1) {
perror("recv");
exit(1);
}
if ((i=recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
perror("recv");
exit(1);
}
DIWrite(&DocumentIndexPost,DocID,packedHedder.subname, NULL);
//printf("DIWrite: %i\n",DocID);
}
else if (packedHedder.command == C_DIRead) {
int DocID;
struct DocumentIndexFormat DocumentIndexPost;
printf("got commane C_DIRead. sise %i hsize %i ds %i\n",packedHedder.size, sizeof(packedHedder), sizeof(DocID));
if ((i=recv(socket, &DocID, sizeof(DocID),0)) == -1) {
perror("recv");
exit(1);
}
//printf("DocID %i\n",DocID);
//leser inn datan
//int DIRead (struct DocumentIndexFormat *DocumentIndexPost, int DocID);
DIRead(&DocumentIndexPost,DocID,packedHedder.subname);
sendall(socket,&DocumentIndexPost, sizeof(struct DocumentIndexFormat));
}
else if (packedHedder.command == C_rGetIndexTime) {
int Lotnr;
unsigned int IndexTime;
if ((i=recv(socket, &LotNr, sizeof(LotNr),0)) == -1) {
perror("recv");
exit(1);
}
IndexTime = GetLastIndexTimeForLot(LotNr,packedHedder.subname);
sendall(socket,&IndexTime, sizeof(IndexTime));
}
else if (packedHedder.command == C_rSetIndexTime) {
int Lotnr;
if ((i=recv(socket, &LotNr, sizeof(LotNr),0)) == -1) {
perror("recv");
exit(1);
}
setLastIndexTimeForLot(LotNr,NULL,packedHedder.subname);
}
else if (packedHedder.command == C_rSendFile) {
//skal mota en fil for lagring i reposetoryet
//char FilePath[156];
FILE *FILEHANDLER;
char c;
char opentype[2];
//char *filblocbuff;
//off_t fileBloks,filerest;
if ((i=recv(socket, &LotNr, sizeof(LotNr),MSG_WAITALL)) == -1) {
perror("Cant recv lotnr");
exit(1);
}
printf("lotNr %i\n",LotNr);
//leser destinasjonelengden
if ((i=recv(socket, &destLeng, sizeof(destLeng),MSG_WAITALL)) == -1) {
perror("Cant recv destLeng");
exit(1);
}
if (destLeng > sizeof(dest)) {
printf("dest filname is to long at %i\n",destLeng);
exit(1);
}
//leser destinasjonene
if ((i=recv(socket, &dest, destLeng,MSG_WAITALL)) == -1) {
perror("Cant recv dest");
exit(1);
}
printf("coping %s as length %i in to lot %i\n",dest,destLeng,LotNr);
if ((i=recv(socket, &opentype, sizeof(char) +1,MSG_WAITALL)) == -1) {
perror("Cant recv opentype");
exit(1);
}
printf("opentype \"%s\"\n",opentype);
//GetFilPathForLot(FilePath,LotNr,packedHedder.subname);
//legger til filnavnet
//strncat(FilePath,dest,sizeof(FilePath));
//leser inn filstørelsen
if ((i=recv(socket, &fileBloks, sizeof(fileBloks),MSG_WAITALL)) == -1) {
perror("Cant recv fileBloks");
exit(1);
}
if ((i=recv(socket, &filerest, sizeof(filerest),MSG_WAITALL)) == -1) {
perror("Cant recv filerest");
exit(1);
}
printf("fileBloks: %" PRId64 ", filerest: %" PRId64 "\n",fileBloks,filerest);
//åpner filen
if ((FILEHANDLER = lotOpenFileNoCasheByLotNr(LotNr,dest,opentype,'e',packedHedder.subname)) == NULL) {
perror(dest);
}
filblocbuff = (char *)malloc(rNetTrabsferBlok);
for(i=0; i < fileBloks; i++) {
if ((n=recv(socket, filblocbuff, rNetTrabsferBlok,MSG_WAITALL)) == -1) {
perror("Cant recv dest");
exit(1);
}
fwrite(filblocbuff,sizeof(c),rNetTrabsferBlok,FILEHANDLER);
}
printf("did recv %i fileBloks\n",i);
if ((n=recv(socket, filblocbuff, filerest,MSG_WAITALL)) == -1) {
perror("Cant recv filerest");
exit(1);
}
fwrite(filblocbuff,sizeof(c),filerest,FILEHANDLER);
free(filblocbuff);
fclose(FILEHANDLER);
printf("\n");
}
else if (packedHedder.command == C_DIGetIp) {
unsigned int DocID;
struct DocumentIndexFormat DocumentIndexPost;
//printf("got command C_DIGetIp\n");
if ((i=recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
perror("recv");
exit(1);
}
//printf("DocID %u\n",DocID);
DIRead(&DocumentIndexPost,DocID,packedHedder.subname);
//printf("ipadress: %u\n",DocumentIndexPost.IPAddress);
sendall(socket,&DocumentIndexPost.IPAddress, sizeof(DocumentIndexPost.IPAddress));
}
else if (packedHedder.command == C_anchorAdd) {
size_t textlen;
unsigned int DocID;
char *text;
printf("Add anchor....\n");
if ((i = recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
perror("recv");
exit(1);
} else if ((i = recv(socket, &textlen, sizeof(textlen), MSG_WAITALL)) == -1) {
perror("recv(textlen)");
exit(1);
}
text = malloc(textlen+1);
text[textlen] = '\0';
if ((i = recv(socket, text, textlen, MSG_WAITALL)) == -1) {
perror("recv(text)");
exit(1);
}
anchoraddnew(DocID, text, textlen, packedHedder.subname, NULL);
printf("Text for %d: %s\n", DocID, text);
free(text);
}
else if (packedHedder.command == C_anchorGet) {
size_t len;
char *text;
int LotNr;
unsigned int DocID;
printf("Get anchor...\n");
if ((i = recv(socket, &DocID, sizeof(DocID),MSG_WAITALL)) == -1) {
perror("recv");
exit(1);
}
printf("got DocID %u\n",DocID);
LotNr = rLotForDOCid(DocID);
printf("trying to read anchor\n");
len = anchorRead(LotNr, packedHedder.subname, DocID, NULL, -1);
printf("got anchor of length %i\n",len);
sendall(socket, &len, sizeof(len));
text = malloc(len+1);
printf("readint it again\n");
anchorRead(LotNr, packedHedder.subname, DocID, text, len+1);
sendall(socket, text, len);
}
else if (packedHedder.command == C_readHTML) {
/*
unsigned int DocID;
unsigned int len;
char *text;
char *acla, *acld;
struct DocumentIndexFormat DocIndex;
struct ReposetoryHeaderFormat ReposetoryHeader;
if ((i = recv(socket, &DocID, sizeof(DocID), MSG_WAITALL)) == -1) {
perror("recv");
exit(1);
}
if ((i = recv(socket, &len, sizeof(len), MSG_WAITALL)) == -1) {
perror("recv(len)");
exit(1);
}
printf("len %u\n",len);
text = malloc(len);
if (text == NULL)
exit(1);
DIRead(&DocIndex, DocID, packedHedder.subname);
if (!rReadHtml(
text,
&len,
DocIndex.RepositoryPointer,
DocIndex.htmlSize,
DocID,
packedHedder.subname,
&ReposetoryHeader,
&acla,
&acld,
DocIndex.imageSize)) {
len = 0;
sendall(socket, &len, sizeof(len));
} else {
++len; // \0
#ifdef DEBUG
printf("docID %u\n",DocID);
printf("Got: (len %i, real %i) ########################\n%s\n#####################\n", len, strlen(text), text);
#endif
sendall(socket, &len, sizeof(len));
sendall(socket, text, len);
sendall(socket, &ReposetoryHeader,sizeof(ReposetoryHeader));
}
free(text);
*/
}
/*
runarb: 06 des 2007: vi har gåt bort fra denne metoden for nå, og bruker heller index over smb. Men tar vare på den da vi kan trenge den siden
else if (packedHedder.command == C_urltodocid) {
char cmd;
int alloclen;
char *urlbuf;
if (urltodociddb == NULL) {
cmd = C_DOCID_NODB;
sendall(socket, &cmd, sizeof(cmd));
exit(1);
} else {
cmd = C_DOCID_READY;
sendall(socket, &cmd, sizeof(cmd));
}
cmd = C_DOCID_NEXT;
alloclen = 1024;
urlbuf = malloc(alloclen);
do {
unsigned int DocID;
size_t len;
if ((i = recv(socket, &cmd, sizeof(cmd), MSG_WAITALL)) == -1) {
err(1, "recv(cmd)");
}
if (cmd == C_DOCID_DONE)
break;
if ((i == recv(socket, &len, sizeof(len), MSG_WAITALL)) == -1) {
err(1, "recv(len)");
}
if (alloclen < len+1) {
free(urlbuf);
alloclen *= 2;
urlbuf = malloc(alloclen);
}
if ((i == recv(socket, urlbuf, len, MSG_WAITALL)) == -1) {
err(1, "recv(len)");
}
urlbuf[len] = '\0';
if (!getDocIDFromUrl(urltodociddb, urlbuf, &DocID)) {
cmd = C_DOCID_NOTFOUND;
sendall(socket, &cmd, sizeof(cmd));
} else {
cmd = C_DOCID_FOUND;
sendall(socket, &cmd, sizeof(cmd));
sendall(socket, &DocID, sizeof(DocID));
}
} while (1);
free(urlbuf);
}
*/
else {
printf("unnown comand. %i\n", packedHedder.command);
}
//printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
} //while
}
int isOkCrawled (char subname[], struct gcaoptFormat *gcaopt) {
char mysql_query [2048];
static MYSQL demo_db;
MYSQL_RES *mysqlres; /* To be used to fetch information into */
MYSQL_ROW mysqlrow;
int nrofcollections;
int i,y;
int crawler_success = 0;
mysql_init(&demo_db);
//koble til mysql
if(!mysql_real_connect(&demo_db, MYSQL_HOST, MYSQL_USER, MYSQL_PASS, BOITHO_MYSQL_DB, 3306, NULL, 0)){
printf(mysql_error(&demo_db));
blog(gcaopt->logSummary,1,"MySQL Error: \"%s\".",mysql_error(&demo_db));
exit(1);
}
sprintf(mysql_query, "select crawler_success from shares where collection_name='%s'",subname);
debug("mysql_query: %s\n",mysql_query);
if(mysql_real_query(&demo_db, mysql_query, strlen(mysql_query))){ /* Make query */
printf(mysql_error(&demo_db));
blog(gcaopt->logSummary,1,"MySQL Error: \"%s\".",mysql_error(&demo_db));
exit(1);
}
mysqlres=mysql_store_result(&demo_db); /* Download result from server */
nrofcollections = (int)mysql_num_rows(mysqlres);
if (nrofcollections == 0) {
blog(gcaopt->logSummary,1,"Skipping \"%s\": dident find any rows",subname);
}
else if (nrofcollections != 1) {
blog(gcaopt->logSummary,1,"Skipping \"%s\": has more then 1 rows",subname);
}
else {
while ((mysqlrow=mysql_fetch_row(mysqlres)) != NULL) { /* Get a row from the results */
if (mysqlrow[0] != NULL) {
crawler_success = atoi(mysqlrow[0]);
}
}
}
mysql_free_result(mysqlres);
mysql_close(&demo_db);
/* Fall back to fullyCrawled if mysql did not contain collection information */
if (nrofcollections == 0) {
char path[PATH_MAX];
int fd;
fprintf(stderr, "Falling back to using fullyCrawled file.\n");
GetFilPathForLot(path, 1, subname);
strcat(path, "fullyCrawled");
fd = open(path, O_RDONLY);
if (fd == -1) {
crawler_success = 0;
} else {
crawler_success = 1;
close(fd);
}
}
printf("~isOkCrawled(crawler_success=%i)\n",crawler_success);
if (crawler_success == 1) {
return 1;
}
else {
return 0;
}
}
void adultWeightopenMemArray(char servername[],char subname[]) {
FILE *FH;
int i;
int LocalLots;
char LotFile[128];
int branksize;
off_t totsize = 0;
struct stat inode;
lotlistLoad();
lotlistMarkLocals(servername);
//aloker minne for rank for de forskjelige lottene
LocalLots=0;
for (i=0;i<maxLots;i++) {
//sjekekr om dette er en lokal lot
if (lotlistIsLocal(i)) {
GetFilPathForLot(LotFile,i,subname);
strcat(LotFile,"AdultWeight");
// prøver å opne
if ( (FH = fopen(LotFile,"r+b")) == NULL ) {
perror(LotFile);
adultWeightMemArray[i] = 0;
}
else {
#ifdef DEBUG
printf("loaded lot %i\n",i);
#endif
branksize = sizeof(unsigned char) * NrofDocIDsInLot;
#ifdef MMAP_ADULT
fstat(fileno(FH),&inode);
if (inode.st_size < branksize) {
fprintf(stderr,"adultWeightopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize);
/*
Stretch the file size to the size of the (mmapped) array of ints
*/
if (fseek(FH, branksize +1, SEEK_SET) == -1) {
perror("Error calling fseek() to 'stretch' the file");
exit(EXIT_FAILURE);
}
/* Something needs to be written at the end of the file to
* have the file actually have the new size.
* Just writing an empty string at the current file position will do.
*
* Note:
* - The current position in the file is at the end of the stretched
* file due to the call to fseek().
* - An empty string is actually a single '\0' character, so a zero-byte
* will be written at the last byte of the file.
*/
if (fwrite("", 1, 1, FH) != 1) {
perror("Error writing last byte of the file");
exit(EXIT_FAILURE);
}
}
if ((adultWeightMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == NULL) {
fprintf(stderr,"adultWeightopenMemArray: can't mmap for lot %i\n",i);
perror("mmap");
}
#else
if ((adultWeightMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) {
printf("malloc eror for lot %i\n",i);
perror("malloc");
exit(1);
}
fread(adultWeightMemArray[i],branksize,1,FH);
#endif
//debug: viser alle rankene vi laster
//for(y=0;y<branksize;y++) {
// printf("DocID %i, rank %i\n",y,adultWeightMemArray[i][y]);
//}
fclose(FH);
totsize += branksize;
++LocalLots;
}
}
else {
adultWeightMemArray[i] = 0;
}
}
printf("adultWeightopenMemArray: have %i local lots\n",LocalLots);
printf("adultWeightopenMemArray: loaded a total of %"PRId64" bytes\n",totsize);
}
main (int argc, char *argv[]) {
char lotpath[512];
int lotNr;
char file[512];
char *optHost = NULL;
extern char *optarg;
extern int optind, opterr, optopt;
char c;
while ((c=getopt(argc,argv,"h:"))!=-1) {
switch (c) {
case 'h':
optHost = optarg;
fprintf(stderr, "Will send to host %s\n",optHost);
break;
default:
exit(1);
}
}
--optind;
if (argc -optind < 3) {
printf("Dette programet kopierer en lot til en annen server\n\n\t./lotcp lotnr subname [file]\n\n");
printf("Options:\n\t-h host\n\n");
exit(0);
}
lotNr = atol(argv[1 +optind]);
char *subname = argv[2 +optind];
//int rmkdir(char dest[], int LotNr,char subname[]);
rmkdir("",lotNr,subname);
//void GetFilPathForLot(char *FilePath,int LotNr,char subname[]);
GetFilPathForLot(lotpath,lotNr,argv[2 +optind]);
#ifdef DEBUG
printf("lotpath %s\n",lotpath);
#endif
if ((argc -optind) == 3) {
recursiveDir(lotpath,"",lotNr,subname,optHost);
}
else if ((argc -optind) == 4) {
strcpy(file,argv[3 +optind]);
//hvis vi er en mappe må vi slutte på /. Hvis vi ikke her det legger vi til en /
if((strchr(file,'/') != NULL) && (file[strlen(file) -1] != '/')) {
printf("adding a / to \"%s\"\n",file);
strcat(file,"/");
}
recursiveDir(lotpath,file,lotNr,subname,optHost);
}
else {
fprintf(stderr,"wrong argc count.\n");
}
}
//gir andre tilgan til lot filer. Casher opne filhandlere
FILE *lotOpenFile(unsigned int DocID,char resource[],char type[], char lock,char subname[]) {
int LotNr;
int i;
char FilePath[128];
char File [128];
if (!LotFilesInalisert) {
for(i=0; i < MaxOpenFiles; i++) {
OpenFiles[i].LotNr = -1;
}
LotFilesInalisert = 1;
}
File[0] = '\0';
//finner i hvilken lot vi skal lese fra
LotNr = rLotForDOCid(DocID);
//printf("LotNr: %i, DocID: %i\n",LotNr,DocID);
//begynner med å søke cashen. Lopper til vi enten er ferdig, eller til vi har funne ønskede i cashen
i = 0;
while ((i < MaxOpenFiles) && (OpenFiles[i].LotNr != LotNr)) {
i++;
}
//temp: skrur av søking her med i=0
//type of og subname er også lagt til uten at det tar hensyn til det i søket
i = 0;
//hvis vi fant i casehn returnerer vi den
if (OpenFiles[i].LotNr == LotNr
&& (strcmp(OpenFiles[i].subname,subname) == 0)
&& (strcmp(OpenFiles[i].type,type)==0)
&& (strcmp(OpenFiles[i].resource,resource)==0)
) {
#ifdef DEBUG
printf("lotOpenFile: fant en tildigere åpnet fil, returnerer den.\n");
printf("lotOpenFile: returnerer: i %i, subname \"%s\", type \"%s\", LotNr %i\n",i,OpenFiles[i].subname,OpenFiles[i].type,OpenFiles[i].LotNr);
printf("lotOpenFile: file is \"%s\"\n",OpenFiles[i].filename);
printf("lotOpenFile: returning file handler %p\n",OpenFiles[i].FILEHANDLER);
#endif
if (OpenFiles[i].FILEHANDLER == NULL) {
printf("Error: FILEHANDLER is NULL\n");
#ifdef DEBUG
exit(-1);
#endif
}
return OpenFiles[i].FILEHANDLER;
}
//hvis ikke åpner vi og returnerer
else {
//hvis dette er en åpen filhånterer, må vi lukke den
if (OpenFiles[i].LotNr != -1) {
printf("lotOpenFile: closeing: i %i\n",i);
fclose(OpenFiles[i].FILEHANDLER);
OpenFiles[i].LotNr = -1;
}
if ((OpenFiles[i].FILEHANDLER = lotOpenFileNoCasheByLotNr( LotNr, resource,type, lock,subname)) == NULL) {
printf("lotOpenFileNoCashe: can't open file\n");
return NULL;
}
GetFilPathForLot(FilePath,LotNr,subname);
strscpy(File,FilePath,sizeof(File));
strlcat(File,resource,sizeof(File));
strscpy(OpenFiles[i].filename,File,sizeof(OpenFiles[i].filename));
strscpy(OpenFiles[i].resource,resource,sizeof(OpenFiles[i].resource));
strscpy(OpenFiles[i].subname,subname,sizeof(OpenFiles[i].subname));
strscpy(OpenFiles[i].type,type,sizeof(OpenFiles[i].type));
//#ifdef DEBUG
printf("lotOpenFile: opening file \"%s\" for %s\n",File,type);
//#endif
OpenFiles[i].LotNr = LotNr;
return OpenFiles[i].FILEHANDLER;
}
}
int lotOpenFileNoCasheByLotNrl(int LotNr,char resource[],char type[], char lock,char subname[]) {
int fd;
int i;
char FilePath[PATH_MAX]; //var 128
char File [PATH_MAX]; //var 128
#ifdef DEBUG
printf("lotOpenFileNoCasheByLotNrl(LotNr=%i, subname=\"%s\", resource=\"%s\")\n",LotNr,subname,resource);
#endif
GetFilPathForLot(FilePath,LotNr,subname);
strcpy(File,FilePath);
strncat(File,resource,PATH_MAX); //var 128
#ifdef DEBUG
printf("lotOpenFileNoCasheByLotNr: opening file \"%s\" for %s\n",File,type);
#endif
if (strcmp(type,">>") == 0) {
//emulating perl's >>. If the file eksist is is opene for reading and writing.
//if not it is createt and openf for writing and reading
if ( (fd = open64(File,O_CREAT|O_RDWR,0664)) == -1 ) {
makePath(FilePath);
if ( (fd = open64(File,O_CREAT|O_RDWR,0664)) == -1 ) {
perror(File);
return -1;
}
}
}
//hvis dette er lesing så hjelper det ikke og prøve å opprette path. Filen vil fortsatt ikke finnes
else if ((strcmp(type,"rb") == 0) || (strcmp(type,"r") == 0)) {
if ( (fd = open64(File,O_RDONLY)) == -1 ) {
#ifdef DEBUG
perror(File);
#endif
return -1;
}
}
//hvis dette er lesing så hjelper det ikke og prøve å opprette path. Filen vil fortsatt ikke finnes
else if ((strcmp(type,"r+b") == 0) || (strcmp(type,"r+") == 0)) {
if ( (fd = open64(File,O_RDWR)) == -1 ) {
#ifdef DEBUG
perror(File);
#endif
return -1;
}
}
else {
fprintf(stderr,"lotOpenFileNoCasheByLotNrl: ikke implementert\n");
exit(1);
/*
//temp: Bytte ut FilePath med filnavnet
if ( (fd = open64(File,type)) == NULL ) {
makePath(FilePath);
//hvorfår har vi type "File" her ???, det verste er at det ser ut til å fungere også
//if ( (FILEHANDLER = (FILE *)fopen64(File,"File")) == NULL ) {
if ( (fd = (open64(File,type)) == NULL ) {
perror(File);
//exit(0);
return NULL;
}
}
*/
}
#ifdef DEBUG
printf("lotOpenFile: tryint to obtain lock \"%c\"\n",lock);
#endif
//honterer låsning
if (lock == 'e') {
//skal vi ha flock64() her ?
flock(fd,LOCK_EX);
}
else if (lock == 's') {
flock(fd,LOCK_SH);
}
#ifdef DEBUG
printf("lotOpenFile: lock obtained\n");
#endif
#ifdef DEBUG
printf("lotOpenFileNoCasheByLotNr: finished\n");
#endif
return fd;
}
void adultWeightopenMemArray2(char subname[]) {
FILE *FH;
int i, y ,z;
int LocalLots;
char LotFile[128];
int branksize;
off_t totsize = 0;
struct stat inode;
int locklimit;
//finner grensen på antal sider vi kan låse i minne
struct rlimit rlim;
if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
printf("Warning: Cannot raise RLIMIT_MEMLOCK limits.");
locklimit = 0;
}
else {
locklimit = rlim.rlim_cur;
}
//aloker minne for rank for de forskjelige lottene
LocalLots=0;
for (i=0;i<maxLots;i++) {
//sjekekr om dette er en lokal lot
adultWeightMemArray[i] = 0;
GetFilPathForLot(LotFile,i,subname);
strcat(LotFile,"AdultWeight");
// prøver å opne
if ( (FH = fopen(LotFile,"r+b")) == NULL ) {
//perror(LotFile);
continue;
}
fstat(fileno(FH),&inode);
if (inode.st_size == 0) {
printf("adultWeightopenMemArray2: file is emty\n");
continue;
}
//#ifdef DEBUG
printf("loaded lot %i\n",i);
//#endif
branksize = sizeof(unsigned char) * NrofDocIDsInLot;
#ifdef MMAP_ADULT
if (inode.st_size < branksize) {
fprintf(stderr,"adultWeightopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize);
/*
Stretch the file size to the size of the (mmapped) array of ints
*/
if (fseek(FH, branksize +1, SEEK_SET) == -1) {
perror("Error calling fseek() to 'stretch' the file");
exit(EXIT_FAILURE);
}
/* Something needs to be written at the end of the file to
* have the file actually have the new size.
* Just writing an empty string at the current file position will do.
*
* Note:
* - The current position in the file is at the end of the stretched
* file due to the call to fseek().
* - An empty string is actually a single '\0' character, so a zero-byte
* will be written at the last byte of the file.
*/
if (fwrite("", 1, 1, FH) != 1) {
perror("Error writing last byte of the file");
exit(EXIT_FAILURE);
}
}
if ((adultWeightMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == NULL) {
fprintf(stderr,"adultWeightopenMemArray: can't mmap for lot %i\n",i);
perror("mmap");
}
//hvis vi kan låse uendelig med minne gjør vi det
if (locklimit == -1) {
//laster all rankerings dataen fra minne, slik ad det går fort å leste den inn siden
z = 0;
for(y=0;y<NrofDocIDsInLot;y++) {
z += adultWeightMemArray[i][y];
}
//låser minne
if (mlock(adultWeightMemArray[i],branksize) != 0) {
perror("mlock");
//exit(1);
}
}
#else
if ((adultWeightMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) {
printf("malloc eror for lot %i\n",i);
perror("malloc");
exit(1);
}
fread(adultWeightMemArray[i],branksize,1,FH);
#endif
//debug: viser alle rankene vi laster
//for(y=0;y<branksize;y++) {
// printf("DocID %i, rank %i\n",y,adultWeightMemArray[i][y]);
//}
fclose(FH);
totsize += branksize;
++LocalLots;
}
printf("adultWeightopenMemArray: have %i local lots\n",LocalLots);
printf("adultWeightopenMemArray: loaded a total of %"PRId64" bytes\n",totsize);
}
int main (int argc, char *argv[]) {
int lotNr;
int lotPart;
char path[256];
char iipath[256];
unsigned lastIndexTime;
int optMustBeNewerThen = 0;
int optAllowDuplicates = 0;
struct revIndexArrayFomat *revIndexArray;
revIndexArray = malloc(sizeof(struct revIndexArrayFomat) * revIndexArraySize);
extern char *optarg;
extern int optind, opterr, optopt;
char c;
while ((c=getopt(argc,argv,"nd"))!=-1) {
switch (c) {
case 'n':
optMustBeNewerThen = 1;
break;
case 'd':
optAllowDuplicates = 1;
break;
case 'v':
break;
default:
exit(1);
}
}
--optind;
printf("lot %s, %i\n",argv[1],argc);
char *type = argv[1 +optind];
lotNr = atoi(argv[2 +optind]);
char *subname = argv[3 +optind];
if ((argc -optind)== 4) {
//finner siste indekseringstid
lastIndexTime = GetLastIndexTimeForLot(lotNr,subname);
if(lastIndexTime == 0) {
printf("lastIndexTime is 0\n");
exit(1);
}
//sjekker om vi har nokk palss
if (!lotHasSufficientSpace(lotNr,4096,subname)) {
printf("insufficient disk space\n");
exit(1);
}
printf("Indexing all buvkets for lot %i\n",lotNr);
for (lotPart=0;lotPart<64;lotPart++) {
//printf("indexint part %i for lot %i\n",lotPart,lotNr);
//"$revindexPath/$revindexFilNr.txt";
GetFilPathForLot(path,lotNr,subname);
//ToDo: må sette språk annen plass
sprintf(iipath,"%siindex/%s/index/aa/",path,argv[1 +optind]);
//oppretter paths
makePath(iipath);
sprintf(iipath,"%s%i.txt",iipath,lotPart);
if ((optMustBeNewerThen != 0)) {
if (fopen(iipath,"r") != NULL) {
printf("we all redy have a iindex.\n");
continue;
}
}
Indekser(iipath,revIndexArray,lotNr,type,lotPart,subname,optAllowDuplicates);
}
}
else if ((argc - optind) == 5) {
lotPart = atoi(argv[4 +optind]);
printf("indexint part %i for lot %i\n",lotPart,lotNr);
//"$revindexPath/$revindexFilNr.txt";
GetFilPathForLot(path,lotNr,subname);
//ToDo: må sette språk annen plass
//aa sprintf(iipath,"%siindex/%s/index/aa/%i.txt",path,argv[1 +optind],lotPart);
//ToDo: må sette språk annen plass
sprintf(iipath,"%siindex/%s/index/aa/",path,argv[1 +optind]);
//oppretter paths
makePath(iipath);
sprintf(iipath,"%s%i.txt",iipath,lotPart);
printf("iipath: \"%s\n",iipath);
if ((optMustBeNewerThen != 0)) {
if (fopen(iipath,"r") != NULL) {
printf("we all redy have a iindex.\n");
exit(1);
}
}
Indekser(iipath,revIndexArray,lotNr,type,lotPart,subname,optAllowDuplicates);
}
else {
printf("usage: ./LotInvertetIndexMaker type lotnr subname [ lotPart ]\n\n");
}
//GetFilPathForLot(lotNr);
}
int
gcrepo(int LotNr, char *subname)
{
int i;
struct ReposetoryHeaderFormat ReposetoryHeader;
char htmlbuffer[524288];
char imagebuffer[524288];
char *acl_allow;
char *acl_deny;
char *url, *attributes;
unsigned long int raddress;
char path[1024];
char path2[1024];
char path3[1024];
FILE *FNREPO;
struct reformat *re;
int keept = 0;
int gced = 0;
container *attrkeys = ropen();
if((re = reopen(LotNr, sizeof(struct DocumentIndexFormat), "DocumentIndex", subname, RE_HAVE_4_BYTES_VERSION_PREFIX|RE_COPYONCLOSE)) == NULL) {
perror("reopen DocumentIndex");
return 0;
}
if ( (FNREPO = lotOpenFileNoCasheByLotNr(LotNr,"reposetory","rb", 's',subname)) == NULL) {
#ifdef DEBUG
printf("lot dont have a reposetory file\n");
#endif
return 0;
}
while (rGetNext_fh(LotNr,&ReposetoryHeader,htmlbuffer,sizeof(htmlbuffer),imagebuffer,&raddress,0,0,subname,&acl_allow,&acl_deny, FNREPO ,&url, &attributes)) {
#ifdef DEBUG
printf("dokument \"%s\", DocID %u.\n",
RE_DocumentIndex(re,ReposetoryHeader.DocID)->Url,
ReposetoryHeader.DocID);
#endif
//printf("%p\n", docindex.RepositoryPointer);
if (raddress != RE_DocumentIndex(re,ReposetoryHeader.DocID)->RepositoryPointer) {
#ifdef DEBUG
printf("Garbage collecting %d at %u. docindex has %u\n", ReposetoryHeader.DocID, raddress,RE_DocumentIndex(re,ReposetoryHeader.DocID)->RepositoryPointer);
#endif
++gced;
}
else {
unsigned long int offset;
offset = rApendPost(&ReposetoryHeader, htmlbuffer, imagebuffer, subname, acl_allow, acl_deny, "repo.wip", url, attributes, attrkeys);
RE_DocumentIndex(re,ReposetoryHeader.DocID)->RepositoryPointer = offset;
#ifdef DEBUG
printf("Writing DocID: %d\n", ReposetoryHeader.DocID);
#endif
++keept;
}
}
fclose(FNREPO);
//lokker filen repo.wip
//lotCloseFiles();
rclose(attrkeys);
printf("keept %i\ngced %i\n",keept,gced);
reclose(re);
/* And we have a race... */
GetFilPathForLot(path, LotNr, subname);
strcpy(path2, path);
strcpy(path3, path);
strcat(path, "repo.wip");
strcat(path2, "reposetory");
rename(path, path2);
strcpy(path, path3);
strcat(path, "DocumentIndex.wip");
strcat(path3, "DocumentIndex");
rename(path, path3);
#ifdef DI_FILE_CASHE
closeDICache();
#endif
return 0;
}
void popopenMemArray(char servername[], char subname[], char rankfile[]) {
FILE *FH;
int i;
int LocalLots;
char LotFile[128];
int branksize;
struct stat inode;
off_t totsize = 0;
lotlistLoad();
lotlistMarkLocals(servername);
//aloker minne for rank for de forskjelige lottene
LocalLots=0;
for (i=0;i<maxLots;i++) {
//sjekekr om dette er en lokal lot
if (lotlistIsLocal(i)) {
GetFilPathForLot(LotFile,i,subname);
strcat(LotFile,rankfile);
// prøver å opne
if ( (FH = fopen(LotFile,"rb")) == NULL ) {
perror(LotFile);
popMemArray[i] = 0;
}
else {
#ifdef DEBUG
printf("loaded lot %i\n",i);
#endif
branksize = sizeof(unsigned char) * NrofDocIDsInLot;
#ifdef MMAP_POP
fstat(fileno(FH),&inode);
if (inode.st_size != branksize) {
fprintf(stderr,"popopenMemArray: file is smaler then size. file size %"PRId64", suposed to be %i\n",inode.st_size,branksize);
continue;
}
if ((popMemArray[i] = mmap(0,branksize,PROT_READ,MAP_SHARED,fileno(FH),0) ) == NULL) {
fprintf(stderr,"popopenMemArray: can't mmap for lot %i\n",i);
perror("popopenMemArray mmap");
}
#else
if ((popMemArray[i] = (unsigned char*)malloc(branksize)) == NULL) {
printf("malloc eror for lot %i\n",i);
perror("malloc");
exit(1);
}
fread(popMemArray[i],branksize,1,FH);
#endif
//debug: viser alle rankene vi laster
/*
for(y=0;y<branksize;y++) {
printf("DocID %i, rank %i\n",y,popMemArray[i][y]);
}
*/
totsize += branksize;
fclose(FH);
++LocalLots;
}
}
else {
popMemArray[i] = 0;
}
}
printf("popopenMemArray: have %i local lots\n",LocalLots);
printf("popopenMemArray: loaded a total of %"PRId64" bytes\n",totsize);
}
void GetFilPathForLotFile(char *FilePath,char lotfile[],int LotNr,char subname[]) {
GetFilPathForLot(FilePath,LotNr,subname);
strcat(FilePath,lotfile);
}
char *returnFilPathForLot(int LotNr,char subname[]) {
static char FilePath[PATH_MAX];
GetFilPathForLot(FilePath,LotNr,subname);
return FilePath;
}
int main (int argc, char *argv[]) {
int lotNr;
int lotPart;
char path[256];
char revpath[256];
char iipath[256];
unsigned lastIndexTime;
struct revIndexArrayFomat *revIndexArray;
revIndexArray = malloc(sizeof(struct revIndexArrayFomat) * revIndexArraySize);
if (argc < 2) {
}
printf("lot %s, %i\n",argv[1],argc);
if (argc == 3) {
lotNr = atoi(argv[2]);
//finner siste indekseringstid
lastIndexTime = GetLastIndexTimeForLot(lotNr,subname);
if(lastIndexTime == 0) {
printf("lastIndexTime is 0\n");
exit(1);
}
//sjekker om vi har nokk palss
if (!lotHasSufficientSpace(lotNr,4096,subname)) {
printf("insufficient disk space\n");
exit(1);
}
printf("Indexing all buvkets for lot %i\n",lotNr);
for (lotPart=0;lotPart<63;lotPart++) {
//printf("indexint part %i for lot %i\n",lotPart,lotNr);
//"$revindexPath/$revindexFilNr.txt";
GetFilPathForLot(path,lotNr,subname);
sprintf(revpath,"%srevindex/%s/%i.txt",path,argv[1],lotPart);
//ToDo: må sette språk annen plass
sprintf(iipath,"%siindex/%s/index/aa/",path,argv[1]);
//oppretter paths
makePath(iipath);
sprintf(iipath,"%s%i.txt",iipath,lotPart);
Indekser(revpath,iipath,revIndexArray);
//sletter revindex. Ingen vits i å ha den fylle opp plass
//remove(revpath);
}
}
else if (argc == 4) {
lotNr = atoi(argv[2]);
lotPart = atoi(argv[3]);
printf("indexint part %i for lot %i\n",lotPart,lotNr);
//"$revindexPath/$revindexFilNr.txt";
GetFilPathForLot(path,lotNr,subname);
sprintf(revpath,"%srevindex/%s/%i.txt",path,argv[1],lotPart);
//ToDo: må sette språk annen plass
sprintf(iipath,"%siindex/%s/index/aa/%i.txt",path,argv[1],lotPart);
Indekser(revpath,iipath,revIndexArray);
}
else {
printf("usage: ./LotInvertetIndexMaker type lotnr [ lotPart ]\n\n");
}
//GetFilPathForLot(lotNr);
}
main (int argc, char *argv[]) {
int LotNr;
char lotPath[255];
char FileName[255];
struct ReposetoryHeaderFormat ReposetoryHeader;
unsigned long int radress;
char htmlbuffer[524288];
Bytef *htmlncompressBuffer;
int htmlncompressBufferSize;
char imagebuffer[524288];
int errornr;
FILE *IMAGEFILE;
FILE *HTMLFILE;
if (argc < 3) {
printf("Error ingen lotnr spesifisert.\n\nEksempel på bruk for å dumpe lot 2:\n\trread 2 /tmp/boitho/\n");
exit(1);
}
LotNr = atoi(argv[1]);
GetFilPathForLot(&lotPath,LotNr);
printf("Opning lot at: %s\n",lotPath);
//henter minne
htmlncompressBuffer = (Bytef *)malloc(2097152);
//loppergjenom alle
while (rGetNext(LotNr,&ReposetoryHeader,htmlbuffer,imagebuffer,&radress, 0, 0)) {
printf("imageSize %i, htmlSize %i\n",ReposetoryHeader.imageSize,ReposetoryHeader.htmlSize);
if (ReposetoryHeader.response == 200) {
//skriver bilde
sprintf(FileName,"%s%i.jpg",argv[2],ReposetoryHeader.DocID);
if ((IMAGEFILE = fopen(FileName,"wb")) == NULL) {
printf("Open file image file");
perror(FileName);
exit(1);
}
fwrite(imagebuffer,ReposetoryHeader.imageSize,1,IMAGEFILE);
fclose(IMAGEFILE);
//skriver html fil
sprintf(FileName,"%s%i.html",argv[2],ReposetoryHeader.DocID);
if ((HTMLFILE = fopen(FileName,"wb")) == NULL) {
printf("Open file image file");
perror(FileName);
exit(1);
}
//dekomprimerer htmlen
//burde ikke hardkode her
htmlncompressBufferSize = 2097152;
//printf("htmlncompressBufferSize: %i, htmlSize: %i\n",htmlncompressBufferSize, ReposetoryHeader.htmlSize);
if ((errornr = uncompress(htmlncompressBuffer,(uLongf *)&htmlncompressBufferSize,htmlbuffer,(uLong)ReposetoryHeader.htmlSize)) != 0) {
printf("Cant uncompress: %i\n",errornr);
}
else {
fwrite(htmlncompressBuffer,htmlncompressBufferSize,1,HTMLFILE);
fclose(HTMLFILE);
}
printf("DocId: %i url: %s\n",ReposetoryHeader.DocID,ReposetoryHeader.url);
//sletter litt slik at vi ser om det blir noen feil i lesingen
ReposetoryHeader.DocID = -1;
}
}
free(htmlncompressBuffer);
}
