/////////////////////////////////////////////////
// EuProjects-Web-Fetch
void TCordisEuProjWebFetch::OnFetch(const int&, const PWebPg& WebPg){
// print url of downloaded page
printf("%s\n", WebPg->GetUrlStr().CStr());
printf("-----------------------\n");
//printf("%s", WebPg->GetHttpHdStr().CStr());
//printf("%s", WebPg->GetHttpBodyAsStr().CStr());
// get current page url & set of outgoing urls with descriptions
TStr WebPgUrlStr=WebPg->GetUrlStr();
TStrKdV DescUrlStrKdV; WebPg->GetOutDescUrlStrKdV(DescUrlStrKdV);
// check current page type and react accordingly
if (WebPgUrlStr.IsPrefix("http://dbs.cordis.lu/fep-cgi/srchidadb?ACTION=D&")){
// if the page represents project description (ACTION=D)
// save the document
TStr HtmlStr=WebPg->GetHttpBodyAsStr();
WebPgUrlStr.Save(*EuProjSOut);
HtmlStr.Save(*EuProjSOut);
EuProjSOut->Flush();
int DescUrlStrKdN=DescUrlStrKdV.SearchForw(TStrKd("NEXT RECORD"));
if (DescUrlStrKdN!=-1){
// fetch next document
FetchDocN++;
FetchUrlStr=
TStr("http://dbs.cordis.lu/fep-cgi/srchidadb?ACTION=D&SESSION=")+
FetchSesIdStr+"&DOC="+TInt::GetStr(FetchDocN);
FetchRetries=0;
FetchUrl(FetchUrlStr);
} else {
printf("*** No forward pointer.\n");
TSysMsg::Quit();
}
// search for forward pointer (to the next project description)
/*int DescUrlStrKdN=DescUrlStrKdV.SearchForw(TStrKd("NEXT RECORD"));
if (DescUrlStrKdN!=-1){
// fetch next project description (usual, most frequent case)
FetchUrlStr=DescUrlStrKdV[DescUrlStrKdN].Dat; FetchRetries=0;
FetchUrl(FetchUrlStr);
} else {
// last project description doesn't include forward pointer
printf("*** No forward pointer.\n");
TSysMsg::Quit();
}*/
} else
if (WebPgUrlStr.IsPrefix("http://dbs.cordis.lu/fep-cgi/srchidadb?ACTION=R&")){
// if the page represents project record-set (ACTION=R)
// take session id
FetchSesIdStr=WebPgUrlStr.GetWcMatch("*SESSION=*&*", 1);
FetchDocN=1;
FetchUrlStr=
TStr("http://dbs.cordis.lu/fep-cgi/srchidadb?ACTION=D&SESSION=")+
FetchSesIdStr+"&DOC="+TInt::GetStr(FetchDocN);
FetchRetries=0;
FetchUrl(FetchUrlStr);
// move to the first project-description-url (first record-set only)
/*int DescUrlStrKdN=0;
while (DescUrlStrKdN<DescUrlStrKdV.Len()){
TStr UrlStr=DescUrlStrKdV[DescUrlStrKdN].Dat; DescUrlStrKdN++;
if (UrlStr.IsPrefix("http://dbs.cordis.lu/fep-cgi/srchidadb?ACTION=D&")){
// fetch first project-description only
FetchUrl(UrlStr);
}
}
if (DescUrlStrKdN>=DescUrlStrKdV.Len()){
// quit downloading if no project descriptions
printf("*** No project descriptions.\n");
TSysMsg::Quit();
}*/
// fetch next index page
/*int DescUrlStrKdN=DescUrlStrKdV.SearchForw(TStrKd("NEXT 20 RECORDS"));
if (DescUrlStrKdN!=-1){
FetchUrlStr=DescUrlStrKdV[DescUrlStrKdN].Dat;
FetchRetries=0;
FetchUrl(FetchUrlStr);
} else {
printf("*** No next 20 records.\n");
}*/
} else {
// get forward pointer to the first project record-set (start only)
int DescUrlStrKdN=DescUrlStrKdV.SearchForw(TStrKd("NEXT 20 RECORDS"));
if (DescUrlStrKdN!=-1){
FetchUrl(DescUrlStrKdV[DescUrlStrKdN].Dat);}
}
}
