void TWebTxtBs::OnFetch(const int& /*FId*/, const PWebPg& WebPg){ if (!IsUrlStr(WebPg->GetUrlStr())){ TMem HttpRespMem; WebPg->GetHttpResp()->GetAsMem(HttpRespMem); TxtBs->AddDocMem(WebPg->GetUrlStr(), HttpRespMem, WebBsFetch->IsIndexTxtBs()); if (WebPg->IsTxt()){ PWebFilter WebFilter=WebBsFetch->GetWebFilter(); PSIn HtmlDocSIn=TStrIn::New(WebPg->GetHttpBodyAsStr()); PHtmlDoc HtmlDoc=THtmlDoc::New(HtmlDocSIn, hdtHRef); TStr BaseUrlStr=WebPg->GetUrlStr(); TStr RelUrlStr; for (int TokN=0; TokN<HtmlDoc->GetToks(); TokN++){ if (WebFilter->IsUrlTok(HtmlDoc->GetTok(TokN), RelUrlStr)){ PUrl Url=TUrl::New(RelUrlStr, BaseUrlStr); if (Url->IsOk(usHttp)){ WebBsFetch->SetUrlAsFinal(Url); if (WebFilter->IsUrlOk(Url)&& (!WebBsFetch->IsUrlEnqueued(Url))&& (!IsUrlStr(Url->GetAsFinalUrlStr()))){ WebBsFetch->AddFetchUrl(Url); } } } } WebBsFetch->GoFetch(); } } }