示例#1
0
void TWebTxtBs::OnFetch(const int& /*FId*/, const PWebPg& WebPg){
  if (!IsUrlStr(WebPg->GetUrlStr())){
    TMem HttpRespMem; WebPg->GetHttpResp()->GetAsMem(HttpRespMem);
    TxtBs->AddDocMem(WebPg->GetUrlStr(), HttpRespMem, WebBsFetch->IsIndexTxtBs());
    if (WebPg->IsTxt()){
      PWebFilter WebFilter=WebBsFetch->GetWebFilter();
      PSIn HtmlDocSIn=TStrIn::New(WebPg->GetHttpBodyAsStr());
      PHtmlDoc HtmlDoc=THtmlDoc::New(HtmlDocSIn, hdtHRef);
      TStr BaseUrlStr=WebPg->GetUrlStr(); TStr RelUrlStr;
      for (int TokN=0; TokN<HtmlDoc->GetToks(); TokN++){
        if (WebFilter->IsUrlTok(HtmlDoc->GetTok(TokN), RelUrlStr)){
          PUrl Url=TUrl::New(RelUrlStr, BaseUrlStr);
          if (Url->IsOk(usHttp)){
            WebBsFetch->SetUrlAsFinal(Url);
            if (WebFilter->IsUrlOk(Url)&&
             (!WebBsFetch->IsUrlEnqueued(Url))&&
             (!IsUrlStr(Url->GetAsFinalUrlStr()))){
              WebBsFetch->AddFetchUrl(Url);
            }
          }
        }
      }
      WebBsFetch->GoFetch();
    }
  }
}