void lotPreOpenStartl(int *preOpen[], char filename[], char subname[], int use) {
int i, n;
char buf[1];
if ((*preOpen = malloc(sizeof(int) * maxLots)) == NULL) {
bblog_errno(ERROR, "malloc preOpen");
exit(-1);
}
for (i=0;i<maxLots;i++) {
(*preOpen)[i] = -1;
#ifndef BLACK_BOKS
if (use) {
(*preOpen)[i] = lotOpenFileNoCasheByLotNrl(i,filename,"rb", 'r',subname);
if ((*preOpen)[i] != -1) {
if ((n=posix_fadvise((*preOpen)[i], 0,0,POSIX_FADV_RANDOM)) == 0) {
}
read((*preOpen)[i],&buf,sizeof(buf));
bblog(INFO, "opening %i, as %i",i,(*preOpen)[i]);
}
}
#endif
}
}
void connectHandler(int socket) {
struct packedHedderFormat packedHedder;
int isAuthenticated = 0;
char tkeyForTest[32];
int i,n;
int intrespons;
int count = 0;
container *attrkeys = NULL;
#ifdef DEBUG_TIME
struct timeval start_time, end_time;
struct timeval tot_start_time, tot_end_time;
gettimeofday(&tot_start_time, NULL);
#endif
ionice_benice();
while ((i=recv(socket, &packedHedder, sizeof(struct packedHedderFormat),MSG_WAITALL)) > 0) {
#ifdef DEBUG
printf("size is: %i\nversion: %i\ncommand: %i\n",packedHedder.size,packedHedder.version,packedHedder.command);
#endif
packedHedder.size = packedHedder.size - sizeof(packedHedder);
if (attrkeys == NULL) {
attrkeys = ropen();
}
if (packedHedder.command == bbc_askToAuthenticate) {
if ((i=recv(socket, tkeyForTest, sizeof(tkeyForTest),MSG_WAITALL)) == -1) {
perror("Cant read tkeyForTest");
exit(1);
}
if (1) {
printf("authenticated\n");
intrespons = bbc_authenticate_ok;
bbdocument_init(NULL);
isAuthenticated = 1;
}
else {
printf("authenticate faild\n");
intrespons = bbc_authenticate_feiled;
}
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else {
if (!isAuthenticated) {
printf("user not autentikated\n");
exit(1);
}
if (packedHedder.command == bbc_docadd) {
#ifdef DEBUG
printf("bbc_docadd\n");
#endif
char *subname,*documenturi,*documenttype,*document,*acl_allow,*acl_denied,*title,*doctype;
char *attributes;
int dokument_size;
unsigned int lastmodified;
#ifdef DEBUG_TIME
gettimeofday(&start_time, NULL);
#endif
//subname
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recvall(socket, subname, intrespons)) == 0) {
perror("Cant read subname");
exit(1);
}
//documenturi
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
documenturi = malloc(intrespons +1);
if ((i=recvall(socket, documenturi, intrespons)) == 0) {
perror("Cant read documenturi");
exit(1);
}
//documenttype
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
documenttype = malloc(intrespons +1);
if ((i=recvall(socket, documenttype, intrespons)) == 0) {
perror("Cant read documenttype");
exit(1);
}
//document
//dokument_size
if ((i=recvall(socket, &dokument_size, sizeof(dokument_size))) == 0) {
perror("Cant read dokument_size");
exit(1);
}
document = malloc(dokument_size +1);
if (dokument_size == 0) {
document[0] = '\0';
}
else {
if ((i=recvall(socket, document, dokument_size)) == 0) {
fprintf(stderr,"Can't read document of size %i\n",dokument_size);
perror("recvall");
exit(1);
}
}
//lastmodified
if ((i=recvall(socket, &lastmodified, sizeof(lastmodified))) == 0) {
perror("Cant read lastmodified");
exit(1);
}
//acl_allow
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
acl_allow = malloc(intrespons +1);
if ((i=recvall(socket, acl_allow, intrespons)) == 0) {
perror("Cant read acl_allow");
exit(1);
}
//acl_denied
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
acl_denied = malloc(intrespons +1);
if ((i=recvall(socket, acl_denied, intrespons)) == 0) {
perror("Cant read acl_denied");
exit(1);
}
//title
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
title = malloc(intrespons +1);
if ((i=recvall(socket, title, intrespons)) == 0) {
perror("Cant read title");
exit(1);
}
//doctype
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
doctype = malloc(intrespons +1);
if ((i=recvall(socket, doctype, intrespons)) == 0) {
perror("Cant read doctype");
exit(1);
}
// Attribute list
if ((i = recvall(socket, &intrespons, sizeof(intrespons))) == 0)
err(1, "Can't receive attribute list len");
attributes = malloc(intrespons +1);
if ((i=recvall(socket, attributes, intrespons)) == 0)
err(1, "Can't receive attribute list");
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: bbdn_docadd recv data time: %f\n",getTimeDifference(&start_time, &end_time));
#endif
printf("\n");
printf("########################################################\n");
printf("Url: %s\n",documenturi);
printf("got subname \"%s\": title \"%s\". Nr %i, dokument_size %i attrib: %s\n",subname,title,count,dokument_size, attributes);
printf("########################################################\n");
printf("calling bbdocument_add():\n");
#ifdef DEBUG_TIME
gettimeofday(&start_time, NULL);
#endif
intrespons = bbdocument_add(subname,documenturi,documenttype,document,dokument_size,lastmodified,acl_allow,acl_denied,title,doctype, attributes, attrkeys);
printf(":bbdocument_add end\n");
printf("########################################################\n");
#ifdef DEBUG_TIME
gettimeofday(&end_time, NULL);
printf("Time debug: bbdn_docadd runing bbdocument_add() time: %f\n",getTimeDifference(&start_time, &end_time));
#endif
free(subname);
free(documenturi);
free(documenttype);
free(document);
free(acl_allow);
free(acl_denied);
free(title);
free(doctype);
free(attributes);
// send status
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_opencollection) {
char *subname;
char path[PATH_MAX];
printf("open collection\n");
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1)
err(1, "Cant read intrespons");
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1)
err(1, "Cant read subname");
GetFilPathForLot(path, 1, subname);
strcat(path, "fullyCrawled");
unlink(path);
free(subname);
}
else if (packedHedder.command == bbc_closecollection) {
printf("closecollection\n");
char *subname;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
bbdocument_close(attrkeys);
attrkeys = NULL;
//toDo må bruke subname, og C ikke perl her
printf("cleanin lots start\n");
char command[PATH_MAX];
snprintf(command,sizeof(command),"perl %s -l -s \"%s\"",bfile("perl/cleanLots.pl"),subname);
printf("running \"%s\"\n",command);
intrespons = system(command);
printf("cleanin lots end\n");
// legger subnamet til listen over ventene subnavn, og huper searchd.
lot_recache_collection(subname);
/* We are done crawling */
{
int fd = lotOpenFileNoCasheByLotNrl(1, "fullyCrawled", ">>", '\0', subname);
if (fd == -1) {
warn("Unable to write fullyCrawled file");
} else {
close(fd);
}
}
free(subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_deleteuri) {
printf("deleteuri\n");
char *subname, *uri;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
uri = malloc(intrespons +1);
if ((i=recv(socket, uri, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read uri");
exit(1);
}
uri[intrespons] = '\0';
printf("going to delete: %s from %s\n", uri, subname);
/* Add docid to the gced file */
{
FILE *fh;
unsigned int DocID, lastmodified;
unsigned int lotNr;
int err = 0;
if (uriindex_get(uri, &DocID, &lastmodified, subname) == 0) {
fprintf(stderr,"Unable to get uri info. uri=\"%s\",subname=\"%s\".",uri,subname);
perror("Unable to get uri info");
err++;
}
if (!err) {
lotNr = rLotForDOCid(DocID);
if ((fh = lotOpenFileNoCasheByLotNr(lotNr,"gced","a", 'e',subname)) == NULL) {
perror("can't open gced file");
err++;
} else {
fwrite(&DocID, sizeof(DocID), 1, fh);
fclose(fh);
}
}
if (!err) {
struct reformat *re;
if((re = reopen(rLotForDOCid(DocID), sizeof(struct DocumentIndexFormat), "DocumentIndex", subname, RE_HAVE_4_BYTES_VERSION_PREFIX)) == NULL) {
perror("can't reopen()");
err++;
} else {
DIS_delete(RE_DocumentIndex(re, DocID));
reclose(re);
}
}
//markerer at den er skitten
if (!err) {
FILE *dirtfh;
dirtfh = lotOpenFileNoCashe(DocID,"dirty","ab",'e',subname);
fwrite("1",1,1,dirtfh);
fclose(dirtfh);
}
if (err == 0)
bbdocument_delete(uri, subname);
}
free(subname);
intrespons = 1; // Always return ok for now
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
}
else if (packedHedder.command == bbc_deletecollection) {
printf("deletecollection\n");
char *subname, *uri;
//subname
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
printf("going to delete collection: %s\n", subname);
intrespons = bbdocument_deletecoll(subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
free(subname);
}
else if (packedHedder.command == bbc_addwhisper) {
whisper_t add;
char *subname;
if ((i=recv(socket, &intrespons, sizeof(intrespons),MSG_WAITALL)) == -1)
err(1, "Cant read intrespons");
subname = malloc(intrespons+1);
if ((i=recv(socket, subname, intrespons,MSG_WAITALL)) == -1) {
perror("Cant read subname");
exit(1);
}
subname[intrespons] = '\0';
if ((i=recv(socket, &add, sizeof(add),MSG_WAITALL)) == -1)
err(1, "Cant read add whisper");
gcwhisper_write(subname, add);
free(subname);
}
else if (packedHedder.command == bbc_HasSufficientSpace) {
char *subname;
//subname
if ((i=recvall(socket, &intrespons, sizeof(intrespons))) == 0) {
perror("Cant read intrespons");
exit(1);
}
subname = malloc(intrespons +1);
if ((i=recvall(socket, subname, intrespons)) == 0) {
perror("Cant read subname");
exit(1);
}
// tester bare i lot 1 her. Må også sjekke andre loter når vi begynner å støtte frlere disker på ES.
intrespons = lotHasSufficientSpace(1, 4096, subname);
if ((n=sendall(socket, &intrespons, sizeof(intrespons))) == -1) {
perror("Cant recv filerest");
exit(1);
}
printf("~Asked for HasSufficientSpace for subname \"%s\". Returnerer %d\n",subname, intrespons);
free(subname);
}
else {
printf("unnown comand. %i\n", packedHedder.command);
}
}
++count;
// #ifdef DEBUG_BREAK_AFTER
// if (count >= DEBUG_BREAK_AFTER) {
// printf("exeting after %i docoments\n",count);
// exit(1);
// }
// #endif
}
#ifdef DEBUG_TIME
gettimeofday(&tot_end_time, NULL);
printf("Time debug: bbdn total time time: %f\n",getTimeDifference(&tot_start_time, &tot_end_time));
#endif
}
int lotOpenFileNoCashel(unsigned int DocID,char resource[],char type[], char lock,char subname[]) {
return lotOpenFileNoCasheByLotNrl(rLotForDOCid(DocID),resource,type,lock,subname);
}
struct reformat *reopen(int lotNr, size_t structsize, char file[], char subname[], int flags) {
struct stat inode;
struct reformat *re = NULL;
char openmode[4];
int mmapmode;
int i;
int stretch = 0;
#ifdef DEBUG
printf("reopen(lotNr=%i, structsize=%d, file=%s, subname=%s, flags=%i)\n",lotNr,structsize,file,subname,flags);
#endif
if ((re = malloc(sizeof(struct reformat))) == NULL) {
return NULL;
}
re->flags = flags;
if ((re->flags & RE_HAVE_4_BYTES_VERSION_PREFIX) == RE_HAVE_4_BYTES_VERSION_PREFIX) {
#ifdef DEBUG
printf("reopen: have RE_HAVE_4_BYTES_VERSION_PREFIX\n");
#endif
structsize += 4;
}
if ( ( (re->flags & RE_READ_ONLY) == RE_READ_ONLY ) && ( (re->flags & RE_STRETCH) == RE_STRETCH ) ) {
// skal strekke hvis filen er for liten, men ikke opprette hvis den ikke er der.
strcpy(openmode,"r+b");
mmapmode = PROT_READ;
stretch = 1;
}
else if ( ( (re->flags & RE_READ_ONLY) == RE_READ_ONLY ) && ( (re->flags & RE_CREATE_AND_STRETCH) == RE_CREATE_AND_STRETCH ) ) {
strcpy(openmode,">>");
mmapmode = PROT_READ;
stretch = 1;
}
else if ((re->flags & RE_READ_ONLY) == RE_READ_ONLY) {
strcpy(openmode,"rb");
mmapmode = PROT_READ;
stretch = 0;
}
else {
strcpy(openmode,">>");
mmapmode = PROT_READ|PROT_WRITE;
stretch = 1;
}
re->lotNr = lotNr;
re->structsize = structsize;
re->maxsize = re->structsize * NrofDocIDsInLot;
strcpy(re->mainfile,file);
strcpy(re->subname,subname);
//lager en privat kopi, som vil erstatte orginalen når vi lokker filen.
if ((re->flags & RE_COPYONCLOSE) == RE_COPYONCLOSE) {
printf("Making private copy of file \"%s\"\n",file);
//åpner en temperer fil
sprintf(re->tmpfile,"%s_retmp",re->mainfile);
if((re->fd = lotOpenFileNoCasheByLotNrl(lotNr, re->tmpfile, ">>", 'w', subname)) == -1) {
goto reopen_error;
}
if ((re->fd_tmp = lotOpenFileNoCasheByLotNrl(lotNr, re->mainfile, ">>", 'r', subname)) == -1) {
goto reopen_error;
}
//kopierer tmp til fd
_filecpy(re->fd, re->fd_tmp);
}
else {
if((re->fd = lotOpenFileNoCasheByLotNrl(lotNr, re->mainfile, openmode, 'r', subname)) == -1) {
#ifdef DEBUG
fprintf(stderr,"can't open file %s. for lot %i, subname %s\n",re->mainfile,lotNr,subname);
perror("lot file");
#endif
goto reopen_error;
}
}
fstat(re->fd,&inode);
if ((stretch == 1) && (inode.st_size < re->maxsize) ) {
printf("Stretching re file..\n");
/*
Stretch the file size to the size of the (mmapped) array of ints
*/
if (lseek(re->fd, re->maxsize -1, SEEK_SET) == -1) {
perror("Error calling fseek() to 'stretch' the file");
goto reopen_error;
}
/* Something needs to be written at the end of the file to
* have the file actually have the new size.
* Just writing an empty string at the current file position will do.
*
* Note:
* - The current position in the file is at the end of the stretched
* file due to the call to fseek().
* - An empty string is actually a single '\0' character, so a zero-byte
* will be written at the last byte of the file.
*/
if (write(re->fd, "", 1) != 1) {
perror("Error writing last byte of the file");
goto reopen_error;
}
}
if ((re->mem = mmap(0,re->maxsize,mmapmode,MAP_SHARED,re->fd,0) ) == MAP_FAILED) {
perror("mmap");
//nullsetter dene, slik at det er lettere og se at det er en feil under debugging.
re->mem = NULL;
goto reopen_error;
}
if ((re->flags & RE_POPULATE) == RE_POPULATE) {
int pr = 0;
#ifdef DEBUG
printf("RE_POPULATE'ing\n");
#endif
for(i=0;i<re->maxsize;i++) {
pr += ((char *)re->mem)[i];
}
}
/*
printf("mmap dump:\n");
printf("########################################\n");
int i;
for(i=0;i<1000;i++) {
printf("%c",(re->mem+i)[0]);
//printf("=%c\n",(char)(re->mem + i));
}
printf("########################################\n");
*/
return re;
reopen_error:
if (re != NULL) {
free(re);
}
return NULL;
}
