void TWebPgFetchEvent::OnFetchEnd(const PHttpResp& HttpResp){
EAssert(HttpResp->IsOk());
EndMSecs=TTm::GetCurUniMSecs();
int StatusCd=HttpResp->GetStatusCd();
if (StatusCd/100==2){ // codes 2XX - ok
ChangeLastUrlToLc(HttpResp);
PWebPg WebPg=TWebPg::New(UrlStrV, IpNumV, HttpResp);
WebPg->PutFetchMSecs(GetFetchMSecs());
CloseConn(); Fetch->DisconnUrl(FId);
Fetch->OnFetch(FId, WebPg);
} else
if (StatusCd/100==3){ // codes 3XX - redirection
ChangeLastUrlToLc(HttpResp);
if (UrlStrV.Len()<5){
TStr RedirUrlStr=HttpResp->GetFldVal("Location");
PUrl RedirUrl=TUrl::New(RedirUrlStr, CurUrl->GetUrlStr());
if (RedirUrl->IsOk(usHttp)){
TStrQuV CookieQuV; HttpResp->GetCookieKeyValDmPathQuV(CookieQuV);
AddCookieKeyValDmPathQuV(CookieQuV);
CurUrl=RedirUrl; Retries=0; CloseConn();
TSockHost::GetAsyncSockHost(CurUrl->GetIpNumOrHostNm(), this);
} else {
TStr MsgStr=TStr("Invalid Redirection URL (")+RedirUrlStr+")";
OnFetchError(MsgStr);
}
} else {
TStr MsgStr=TStr("Cycling Redirection [")+TopUrl->GetUrlStr()+"]";
OnFetchError(MsgStr);
}
} else { // all other codes - error
TStr MsgStr=TStr("Http Error (")+
TInt::GetStr(StatusCd)+"/"+HttpResp->GetReasonPhrase()+")";
OnFetchError(MsgStr);
}
}
/////////////////////////////////////////////////
// Web-Net-Proxy-Connection
void TProxyConn::SendHttpResp(TWebNetSrv &WebNetSrv, PHttpResp HttpResp) {
if (WebNetSrv.IsCltSock(RqSockId)) {
//!!warning - two fetch id entries in HTTP header
// rebuild http header to replace 'fetchid'
HttpResp->AddFldVal(THttp::FetchIdFldNm,TInt(RqFetchId).GetStr());
WebNetSrv.SendHttpResp(RqSockId, HttpResp);
//SaveToErrLog((TStr("ProxyRqOk: ")+HttpResp->GetHdStr()).CStr());
}
}
void TWebPgFetchEvent::ChangeLastUrlToLc(const PHttpResp& HttpResp){
static TStr MsNm="Microsoft";
static TStr HttpsNm="HTTPS";
TStr SrvNm=HttpResp->GetSrvNm();
if ((SrvNm.StartsWith(MsNm))||(SrvNm.StartsWith(HttpsNm))){
if (!UrlStrV.Last().IsLc()){
PUrl Url=TUrl::New(UrlStrV.Last());
Url->ToLcPath();
UrlStrV.Last()=Url->GetUrlStr();
}
}
}
void TWebNetClt::OnHttpResp(const PHttpResp& HttpResp){
IAssert(HttpResp->IsOk());
// get fetch-id
TStr FetchIdStr=HttpResp->GetFldVal(THttp::FetchIdFldNm);
int FetchId=FetchIdStr.GetInt(-1);
// check if response has correct fetch-id
if (IsSentQEmpty()){return;} // queue can be emptied by timeout...
int TopFetchId; TTm TopCreateTm; TopFromSentQ(TopFetchId, TopCreateTm);
IAssert(FetchId==TopFetchId);
PopFromSentQ();
// get status-code
int StatusCd=HttpResp->GetStatusCd();
// process http-response
if (StatusCd/100==2){ // codes 2XX - ok
WebNetCltV->OnHttpResp(FetchId, HttpResp);
} else
if (StatusCd/100==3){ // codes 3XX - redirection
TStr RedirUrlStr=HttpResp->GetFldVal("Location");
PUrl RedirUrl=TUrl::New(RedirUrlStr);
if (RedirUrl->IsOk(usHttp)){
//SaveToErrLog((TStr("Redirect: ")+HttpResp->GetHdStr()).CStr());
TStr Host=HttpResp->GetFldVal(THttp::HostFldNm);
WebNetCltV->FetchUrl(RedirUrl, FetchId, Host);
} else {
TStr MsgStr=TStr("Invalid Redirection URL (")+RedirUrlStr+")";
WebNetCltV->OnHttpRespError(FetchId, MsgStr);
}
} else { // all other codes - error
TStr MsgStr=TStr("Http Error (")+
TInt::GetStr(StatusCd)+"/"+HttpResp->GetReasonPhrase()+")";
WebNetCltV->OnHttpRespError(FetchId, MsgStr);
}
}
void TSAppSrvFun::LogReqRes(const TStrKdV& FldNmValPrV, const PHttpResp& HttpResp)
{
if (LogRqFolder == "")
return;
try {
TDir::GenDir(LogRqFolder);
TStr TimeNow = TTm::GetCurLocTm().GetWebLogDateTimeStr(true);
TimeNow.ChangeChAll(':', '.');
PSOut Output = TFOut::New(LogRqFolder + "/" + TimeNow + ".txt", false);
Output->PutStr(FunNm.CStr()); Output->PutCh('\n');
for (int N=0; N < FldNmValPrV.Len(); N++)
Output->PutStrFmt(" %s: %s\n", FldNmValPrV[N].Key.CStr(), FldNmValPrV[N].Dat.CStr());
Output->PutCh('\n');
Output->PutStr(HttpResp->GetBodyAsStr(), false);
} catch (...) {
/*const PNotify& Notify = RqEnv->GetWebSrv()->GetNotify();
Notify->OnStatus("Unable to log request for function '" + GetFunNm() + "'!");*/
}
}
bool IsCTxtHttpResp(const PUrl& Url, const PHttpResp& HttpResp, const int& MnCTxtToks){
if (HttpResp->IsStatusCd_Ok()){
PWebPg WebPg=TWebPg::New(Url->GetUrlStr(), HttpResp);
if (HttpResp->IsContType(THttp::TextHtmlFldVal)){
TMem BodyMem=HttpResp->GetBodyAsMem();
PSIn BodyMemIn=TMemIn::New(BodyMem);
// prepare html-tokens
PHtmlDoc HtmlDoc=THtmlDoc::New(BodyMemIn, hdtAll, false);
int Toks=HtmlDoc->GetToks(); THtmlLxSym TokSym; TStr TokStr;
// prepare continuous-text indicators
int CTxtToks=0; TChA CTxtChA; bool CTxtP=false;
// prepare script & style flag
bool InScript=false; bool InStyle=false;
// traverse tokens
for (int TokN=0; TokN<Toks; TokN++){
// get token data
HtmlDoc->GetTok(TokN, TokSym, TokStr);
switch (TokSym){
case hsyStr:
case hsyNum:
case hsySSym:
if (!InScript&&!InStyle){
// text token
CTxtToks++; CTxtChA+=TokStr; CTxtChA+=' ';
}
break;
case hsyBTag:
if (!InScript&&!InStyle){
if (TokStr=="<SCRIPT>"){
// start of script
InScript=true; CTxtToks=0; CTxtChA.Clr();
} else
if (TokStr=="<STYLE>"){
// start of style
InStyle=true; CTxtToks=0; CTxtChA.Clr();
} else {
if ((TokStr=="<P>")||(TokStr=="<B>")||(TokStr=="<I>")){
// skip in-text-tags
} else {
// non-text-tags - break continuous-text
CTxtToks=0; CTxtChA.Clr();
}
}
}
break;
case hsyETag:
if (InScript||InStyle){
if (TokStr=="<SCRIPT>"){
// end of script
InScript=false;
} else
if (TokStr=="<STYLE>"){
// end of style
InStyle=false;
}
}
break;
default:
// non-text-token - break continuous-text
CTxtToks=0; CTxtChA.Clr();
break;
}
// stop if enough continuous-text
if (CTxtToks>MnCTxtToks){
CTxtP=true; break;
}
}
if (CTxtP){
printf("%s\n", Url->GetUrlStr().CStr());
}
return CTxtP;
}
}
return false;
}
void TWebNetSrv::SendHttpResp(const int& SockId, const PHttpResp& HttpResp){
TMem Mem; HttpResp->GetAsMem(Mem);
PNetObj NetObj=PNetObj(new TNetMem(Mem));
SendNetObj(SockId, NetObj);
}
void TMongCliContext::Respond(const PHttpResp& HttpResp) {
const TMem& Body = HttpResp->GetBodyAsMem();
TStr ResponseHeader = HttpResp->GetHdStr();
mg_write(Conn, ResponseHeader.CStr(), ResponseHeader.Len());
mg_write(Conn, Body.GetBf(), Body.Len());
}
