void linksWrite(struct pagewordsFormat *pagewords,struct addNewUrlhaFormat addNewUrlha[]) {
int i, len;
struct updateFormat updatePost;
for(i=0;i<(*pagewords).nrOfOutLinks;i++) {
if ((IndexerMaxLinks)> i) {
if ((*pagewords).outlinks[i].good) {
len = (*pagewords).outlinks[i].linktextlen;
//fjerner space på slutten
if (len > 0) {
(*pagewords).outlinks[i].linktext[len -1] = '\0';
}
//tømmer minne for å gjøre filene mer komprimeringsvendlige
memset(updatePost.url,'\0',sizeof(updatePost.url));
memset(updatePost.linktext,'\0',sizeof(updatePost.linktext));
strscpy((char *)updatePost.url,(*pagewords).outlinks[i].url,sizeof((*pagewords).outlinks[i].url));
strscpy((char *)updatePost.linktext,(*pagewords).outlinks[i].linktext,sizeof( (*pagewords).outlinks[i].linktext ));
updatePost.DocID_from = (*pagewords).curentDocID;
//bruker sidens DocID til å velge hvilken fil. Usikker på om det er lurt, men det er lett og implementere for nå
addNewUrl(&addNewUrlha[(*pagewords).curentDocID % NEWURLFILES_NR],&updatePost);
}
}
}
}
Spider::Spider(){
page = new WebPage();
page->flag = true;
isRegionNode = false;
connect(page, SIGNAL(addNewUrl(QUrl)), this, SLOT(addUrlToQueue(QUrl)));
server = new QLocalServer();
shared = new QSharedMemory(global::KEY_SHARED_MEMORY,0);
instance = new NetWorker();
}
bool WebPage::acceptNavigationRequest(QWebFrame * frame, const QNetworkRequest & request, NavigationType type)
{
//qDebug()<<"NavigationType :"<<type;
//qDebug()<<"[acceptNavigationRequest] url : "<<request.url();
if(!flag){
emit addNewUrl(request.url());
qDebug()<< "emit singal:"<<request.url();
}
return flag;
}
void linkadd(char word[]) {
struct updateFormat updatePost;
if ((strchr(word,'?') != NULL) && (global_curentUrlIsDynamic)) {
//printf("NO add %s\n",word);
}
else if (strlen(word) > sizeof(updatePost.url)) {
//printf("url to long at %i char\n",strlen(word));
}
else {
//printf("link: %s\n",word);
strncpy(updatePost.url,word,sizeof(updatePost.url));
strncpy(updatePost.linktext,"",sizeof(updatePost.linktext));
updatePost.DocID_from = global_curentDocID;
addNewUrl(&updatePost,"");
}
}
