int main (int argc, char *argv[]) {
struct DocumentIndexFormat DocumentIndexPost;
int PopRankextern;
int PopRankintern;
int PopRanknoc;
int PopRanindex;
char ShortRank;
FILE *FH;
struct popl popextern;
struct popl popintern;
struct popl popnoc;
struct popl popindex;
uLong htmlBufferSize = 0;
char *htmlBuffer = NULL;
char *acl_allowbuffer = NULL;
char *acl_deniedbuffer = NULL;
char timebuf[26];
int optShowhtml = 0;
int optShowWords = 0;
int optSummary = 0;
int optAnchor = 0;
int optResource = 0;
int optPopRank = 0;
int optDelete = 0;
int optAdult = 0;
unsigned int DocID;
char *subname;
if (getenv("QUERY_STRING") == NULL) {
extern char *optarg;
extern int optind, opterr, optopt;
char c;
while ((c=getopt(argc,argv,"hwsarpdu"))!=-1) {
switch (c) {
case 'h':
optShowhtml = 1;
break;
case 'u':
optAdult = 1;
break;
case 'w':
optShowWords = 1;
break;
case 's':
optSummary = 1;
break;
case 'a':
optAnchor = 1;
break;
case 'p':
optPopRank = 1;
break;
case 'r':
optResource = 1;
break;
case 'd':
optDelete = 1;
break;
default:
exit(1);
}
}
--optind;
#ifdef DEBUG
printf("argc %i, optind %i\n",argc,optind);
#endif
if ((argc - optind)!= 3) {
printf("Dette programet gir info om en DocID\n\n\tUsage PageInfo DocID collection\n");
exit(1);
}
DocID = atol(argv[1 +optind]);
subname = argv[2 +optind];
}
else {
printf("Content-type: text/plain\n\n");
int res;
// Initialize the CGI lib
res = cgi_init();
// Was there an error initializing the CGI???
if (res != CGIERR_NONE) {
printf("Error # %d: %s<p>\n", res, cgilib_strerror(res));
fprintf(stderr,"Cgi-lib error.");
return -1;
}
if (cgi_getentrystr("subname") == NULL) {
fprintf(stderr,"Didn't recieve any subname.");
return -1;
}
else {
subname = cgi_getentrystr("subname");
}
if (cgi_getentrystr("DocID") == NULL) {
fprintf(stderr,"Didn't recieve any DocID.");
return -1;
}
else {
DocID = atol( cgi_getentrystr("DocID") );
}
}
html_parser_init();
printf("Showing data for Collection \"%s\", DocID %u\n\n",subname,DocID);
printf("Lot: %i\n",rLotForDOCid(DocID));
if (optDelete) {
memset(&DocumentIndexPost,'\0',sizeof(DocumentIndexPost));
DIWrite(&DocumentIndexPost,DocID,subname,NULL);
return 0;
}
if (DIRead_fmode(&DocumentIndexPost,DocID,subname,'s')) {
printf("Url: \"%s\"\nLanguage: %s (id: %s)\nOffensive code: %hu\nDocument type: %s\nTime tested sins last good crawl: %hu\nAdult weight: %hu\nResource size: %u\nIP Address: %u\nHtml size: %i\nImage size: %i\nUser ID: %i\nCrawler version: %f\nRepository pointer: %u\n",
DocumentIndexPost.Url,
getLangCode2(atoi(DocumentIndexPost.Sprok)),
DocumentIndexPost.Sprok,
DocumentIndexPost.Offensive_code,
DocumentIndexPost.Dokumenttype,
DocumentIndexPost.AntallFeiledeCrawl,
DocumentIndexPost.AdultWeight,
DocumentIndexPost.ResourceSize,
DocumentIndexPost.IPAddress,
DocumentIndexPost.htmlSize2,
DocumentIndexPost.imageSize,
DocumentIndexPost.userID,
DocumentIndexPost.clientVersion,
DocumentIndexPost.RepositoryPointer);
if (DocumentIndexPost.response == 200) {
printf("HTTP response: %hu\n",DocumentIndexPost.response);
}
else {
printf("HTTP response: \033[1;31m%hu\033[0m\n",DocumentIndexPost.response);
}
ctime_r((time_t *)&DocumentIndexPost.CrawleDato,timebuf);
timebuf[24] = '\0';
printf("Last crawled time: %u\n",DocumentIndexPost.CrawleDato);
printf("Last crawled time ISO: %s\n",timebuf);
printf("crc32: %u\n",DocumentIndexPost.crc32);
#ifdef BLACK_BOX
printf("Last seen Unix: %u\n",DocumentIndexPost.lastSeen);
printf("Last seen ISO: %s", ctime(&DocumentIndexPost.lastSeen));
#endif
printf("Nr of out links: %u\n",(unsigned int)DocumentIndexPost.nrOfOutLinks);
char *metadesc, *title, *body;
if (DocumentIndexPost.SummarySize == 0) {
printf("Summary: Don't have pre-parsed summery (summary size is 0)\n");
}
else if (rReadSummary(DocID,&metadesc, &title, &body,DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize,subname)) {
printf("\nSummary:\n");
printf("\tSummary pointer: %u\n\tSummary size: %hu\n",DocumentIndexPost.SummaryPointer,DocumentIndexPost.SummarySize);
printf("\tTitle from summary: \"%s\"\n\tMeta description from summary: \"%s\"\n",title,metadesc);
if (optSummary) {
printf("Summary body\n*******************\n%s\n*******************\n\n",body);
}
}
else {
printf("Don't have pre-parsed summery\n");
}
struct ReposetoryHeaderFormat ReposetoryHeader;
char *url, *attributes;
if (!rReadHtml(&htmlBuffer,&htmlBufferSize,DocumentIndexPost.RepositoryPointer,DocumentIndexPost.htmlSize2,DocID,subname,&ReposetoryHeader,&acl_allowbuffer,&acl_deniedbuffer,DocumentIndexPost.imageSize, &url, &attributes)) {
printf("rReadHtml: did not returne true!\n");
return;
}
printf("Entire url: %s\n", url);
#ifdef BLACK_BOX
printf("acl allow raw: \"%s\"\n",acl_allowbuffer);
printf("acl denied raw: \"%s\"\n",acl_deniedbuffer);
printf("acl allow resolved: \"%s\"\n",aclResolv(acl_allowbuffer));
printf("acl denied resolved: \"%s\"\n",aclResolv(acl_deniedbuffer));
printf("PopRank: %d\n", ReposetoryHeader.PopRank);
#endif
if (optShowhtml) {
printf("html uncompresed size %i\n",htmlBufferSize);
printf("html buff:\n*******************************\n");
fwrite(htmlBuffer,htmlBufferSize,1,stdout);
printf("\n*******************************\n\n");
}
if (optShowWords) {
printf("words:\n");
//run_html_parser( DocumentIndexPost.Url, htmlBuffer, htmlBufferSize, fn );
char *title, *body;
html_parser_run(url,htmlBuffer, htmlBufferSize,&title, &body,fn,NULL );
}
if (optResource) {
char buf[500000];
printf("Resource:\n");
printf("Ptr: 0x%x Len: %x\n", DocumentIndexPost.ResourcePointer, DocumentIndexPost.ResourceSize);
if (getResource(rLotForDOCid(DocID), subname, DocID, buf, sizeof(buf)) == 0) {
printf("\tDid not get any resource\n");
warn("");
} else {
printf("%s\n", buf);
}
}
printf("attributes:\"%s\"\n", attributes);
free(url);
free(attributes);
free(acl_allowbuffer);
free(acl_deniedbuffer);
}
else {
printf("Cant read post\n");
}
#ifndef BLACK_BOX
if (optAdult) {
int httpResponsCodes[nrOfHttpResponsCodes];
//char *title;
//char *body;
struct adultFormat *adult;
struct pagewordsFormat *pagewords = malloc(sizeof(struct pagewordsFormat));
int AdultWeight;
unsigned char langnr;
if ((adult = malloc(sizeof(struct adultFormat))) == NULL) {
perror("malloc argstruct.adult");
exit(1);
}
wordsInit(pagewords);
langdetectInit();
adultLoad(adult);
AdultWeight -1;
handelPage(pagewords,&ReposetoryHeader,htmlBuffer,htmlBufferSize,&title,&body);
wordsMakeRevIndex(pagewords,adult,&AdultWeight,&langnr);
printf("adult %i\n",AdultWeight);
}
if (optAnchor) {
int anchorBufferSize;
char *anchorBuffer;
anchorBufferSize = anchorRead(rLotForDOCid(DocID),subname,DocID,NULL,-1);
anchorBufferSize += 1;
anchorBuffer = malloc(anchorBufferSize);
anchorRead(rLotForDOCid(DocID),subname,DocID,anchorBuffer,anchorBufferSize);
printf("#######################################\nanchors:\n%s\n#######################################\n",anchorBuffer);
free(anchorBuffer);
}
if (optPopRank) {
popopen (&popindex,"/home/boitho/config/popindex");
PopRanindex = popRankForDocID(&popindex,DocID);
popclose(&popindex);
printf("popindex %i\n",PopRanindex);
if (popopen (&popextern,"/home/boitho/config/popextern")) {
PopRankextern = popRankForDocID(&popextern,DocID);
printf("PopRankextern: %i\n",PopRankextern);
popclose(&popextern);
}
if (popopen (&popintern,"/home/boitho/config/popintern")) {
PopRankintern = popRankForDocID(&popintern,DocID);
printf("PopRankintern %i\n",PopRankintern);
popclose(&popintern);
}
if (popopen (&popnoc,"/home/boitho/config/popnoc")) {
PopRanknoc = popRankForDocID(&popnoc,DocID);
printf("PopRanknoc %i\n",PopRanknoc);
popclose(&popnoc);
}
if (popopen (&popindex,"/home/boitho/config/popindex")) {
PopRanindex = popRankForDocID(&popindex,DocID);
printf("popindex %i\n",PopRanindex);
popclose(&popindex);
}
printf("PopRankextern: %i\nPopRankintern %i\nPopRanknoc %i\n",PopRankextern,PopRankintern,PopRanknoc);
int brank;
popopenMemArray_oneLot(subname,rLotForDOCid(DocID));
brank = popRankForDocIDMemArray(DocID);
printf("brank %i\n",brank);
//short rank
if ( (FH = fopen(SHORTPOPFILE,"rb")) == NULL ) {
perror("open");
}
else {
if ((fseek(FH,DocID* sizeof(ShortRank),SEEK_SET) == 0) && (fread(&ShortRank,sizeof(ShortRank),1,FH) != 0)) {
printf("Short rank %u\n",(unsigned char)ShortRank);
}
else {
printf("no hort rank avalibal\n");
};
fclose(FH);
}
} // if optPopRank
#endif
}
int main (int argc, char *argv[]) {
int lotNr;
char lotServer[64];
int pageCount;
int i;
FILE *ADULTWEIGHTFH;
unsigned char awvalue;
unsigned int FiltetTime;
unsigned int FileOffset;
off_t DocIDPlace;
char htmlcompressdbuffer[524288]; //0.5 mb
char imagebuffer[524288]; //0.5 mb
int httpResponsCodes[nrOfHttpResponsCodes];
struct ReposetoryHeaderFormat ReposetoryHeader;
struct DocumentIndexFormat DocumentIndexPost;
unsigned long int radress;
FILE *revindexFilesHa[NrOfDataDirectorys];
struct adultFormat adult;
unsigned int lastIndexTime;
if (argc < 2) {
printf("Dette programet analyserer en url. Usage:\n\tbaddsPageAnalyser url\n");
exit(0);
}
char url[1024];
char *HtmlBuffer = NULL;
adultLoad(&adult);
strncpy(url,argv[1],sizeof(url));
httpGet(&HtmlBuffer,url);
//setter opp en alarm slik at run_html_parser blir avbrut hvis den henger seg
alarm_got_raised = 0;
signal(SIGALRM, html_parser_timout);
alarm( 5 );
//parser htmlen
run_html_parser(url , HtmlBuffer, strlen(HtmlBuffer), fn );
alarm( 0);
if(alarm_got_raised) {
printf("run_html_parser did time out.\n");
}
else {
}
unsigned short AdultWeight;
wordsMakeRevIndex(&adult,&AdultWeight);
//sortere på forekomst
pagewordsSortOnOccurrence();
for(i=0; i<pagewords.revIndexnr; i++) {
printf("%lu forekomster %i (+1) '%s'\n",pagewords.revIndex[i].WordID,pagewords.revIndex[i].nr,pagewords.revIndex[i].word);
}
return 0;
}
void handelPage(char lotServer[], unsigned int LotNr,struct ReposetoryHeaderFormat *ReposetoryHeader, char htmlcompressdbuffer[],char imagebuffebuffer[],FILE *revindexFilesHa[],struct DocumentIndexFormat *DocumentIndexPost, int DocID,int httpResponsCodes[], struct adultFormat *adult) {
unsigned int HtmlBufferSize;
char HtmlBuffer[524288];
int nerror;
//temp: må finne lang
char lang[4] = " aa";
if ((*ReposetoryHeader).response < nrOfHttpResponsCodes) {
++httpResponsCodes[(*ReposetoryHeader).response];
}
//printf("%lu %s\n",(*ReposetoryHeader).DocID, (*ReposetoryHeader).url);
if (((*ReposetoryHeader).response >= 200) && ((*ReposetoryHeader).response <= 299)) {
HtmlBufferSize = sizeof(HtmlBuffer);
if ( (nerror = uncompress((Bytef*)&HtmlBuffer,(uLong *)&HtmlBufferSize,htmlcompressdbuffer,(*ReposetoryHeader).htmlSize)) != 0) {
#ifndef NOWARNINGS
printf("uncompress error. Code: %i\n",nerror);
#endif
}
else {
//if ((*ReposetoryHeader).DocID == 363093661) {
// FILE *FH;
// FH = fopen("/tmp/IndexerLotdump.html","wb");
// fwrite(HtmlBuffer,HtmlBufferSize,1,FH);
// fclose(FH);
// exit(1);
//}
//begynner på en ny side
wordsReset();
if (strcmp((*ReposetoryHeader).content_type,"htm") == 0) {
//setter opp en alarm slik at run_html_parser blir avbrut hvis den henger seg
alarm_got_raised = 0;
signal(SIGALRM, html_parser_timout);
alarm( 5 );
//parser htmlen
run_html_parser( (*ReposetoryHeader).url, HtmlBuffer, HtmlBufferSize, fn );
alarm( 0);
if(alarm_got_raised) {
printf("run_html_parser did time out. At DocID %lu\n",(*ReposetoryHeader).DocID);
}
else {
wordsMakeRevIndex(adult,&(*DocumentIndexPost).AdultWeight);
revindexFilesAppendWords(revindexFilesHa,(*ReposetoryHeader).DocID,lang);
}
}
else if (strcmp((*ReposetoryHeader).content_type,"btx") == 0) {
//ToDo: definer Boitho TeXt formatet
}
else {
printf("unknown content_type \"%s\"\n",(*ReposetoryHeader).content_type);
}
}
}
else { //not in 200->299 range
//radress må vel ikke nulles her, like går å ha den pekene til strukturen i reposetoryet
HtmlBufferSize = 0;
}
}
