예제 #1
0
파일: webpgfetch.cpp 프로젝트: Zala/qminer
void TWebPgFetchEvent::OnFetchEnd(const PHttpResp& HttpResp){
  EAssert(HttpResp->IsOk());
  EndMSecs=TTm::GetCurUniMSecs();
  int StatusCd=HttpResp->GetStatusCd();
  if (StatusCd/100==2){ // codes 2XX - ok
    ChangeLastUrlToLc(HttpResp);
    PWebPg WebPg=TWebPg::New(UrlStrV, IpNumV, HttpResp);
    WebPg->PutFetchMSecs(GetFetchMSecs());
    CloseConn(); Fetch->DisconnUrl(FId);
    Fetch->OnFetch(FId, WebPg);
  } else
  if (StatusCd/100==3){ // codes 3XX - redirection
    ChangeLastUrlToLc(HttpResp);
    if (UrlStrV.Len()<5){
      TStr RedirUrlStr=HttpResp->GetFldVal("Location");
      PUrl RedirUrl=TUrl::New(RedirUrlStr, CurUrl->GetUrlStr());
      if (RedirUrl->IsOk(usHttp)){
        TStrQuV CookieQuV; HttpResp->GetCookieKeyValDmPathQuV(CookieQuV);
        AddCookieKeyValDmPathQuV(CookieQuV);
        CurUrl=RedirUrl; Retries=0; CloseConn();
        TSockHost::GetAsyncSockHost(CurUrl->GetIpNumOrHostNm(), this);
      } else {
        TStr MsgStr=TStr("Invalid Redirection URL (")+RedirUrlStr+")";
        OnFetchError(MsgStr);
      }
    } else {
      TStr MsgStr=TStr("Cycling Redirection [")+TopUrl->GetUrlStr()+"]";
      OnFetchError(MsgStr);
    }
  } else { // all other codes - error
    TStr MsgStr=TStr("Http Error (")+
     TInt::GetStr(StatusCd)+"/"+HttpResp->GetReasonPhrase()+")";
    OnFetchError(MsgStr);
  }
}
/////////////////////////////////////////////////
// Web-Net-Proxy-Connection
void TProxyConn::SendHttpResp(TWebNetSrv &WebNetSrv, PHttpResp HttpResp) {
  if (WebNetSrv.IsCltSock(RqSockId)) {
    //!!warning - two fetch id entries in HTTP header
    // rebuild http header to replace 'fetchid'

    HttpResp->AddFldVal(THttp::FetchIdFldNm,TInt(RqFetchId).GetStr());
    WebNetSrv.SendHttpResp(RqSockId, HttpResp);
    //SaveToErrLog((TStr("ProxyRqOk: ")+HttpResp->GetHdStr()).CStr());
  }
}
예제 #3
0
파일: webpgfetch.cpp 프로젝트: Zala/qminer
void TWebPgFetchEvent::ChangeLastUrlToLc(const PHttpResp& HttpResp){
  static TStr MsNm="Microsoft";
  static TStr HttpsNm="HTTPS";
  TStr SrvNm=HttpResp->GetSrvNm();
  if ((SrvNm.StartsWith(MsNm))||(SrvNm.StartsWith(HttpsNm))){
    if (!UrlStrV.Last().IsLc()){
      PUrl Url=TUrl::New(UrlStrV.Last());
      Url->ToLcPath();
      UrlStrV.Last()=Url->GetUrlStr();
    }
  }
}
void TWebNetClt::OnHttpResp(const PHttpResp& HttpResp){
  IAssert(HttpResp->IsOk());
  // get fetch-id
  TStr FetchIdStr=HttpResp->GetFldVal(THttp::FetchIdFldNm);
  int FetchId=FetchIdStr.GetInt(-1);

  // check if response has correct fetch-id
  if (IsSentQEmpty()){return;} // queue can be emptied by timeout...
  int TopFetchId; TTm TopCreateTm; TopFromSentQ(TopFetchId, TopCreateTm);
  IAssert(FetchId==TopFetchId);
  PopFromSentQ();

  // get status-code
  int StatusCd=HttpResp->GetStatusCd();
  // process http-response
  if (StatusCd/100==2){ // codes 2XX - ok
    WebNetCltV->OnHttpResp(FetchId, HttpResp);
  } else
  if (StatusCd/100==3){ // codes 3XX - redirection
    TStr RedirUrlStr=HttpResp->GetFldVal("Location");
    PUrl RedirUrl=TUrl::New(RedirUrlStr);
    if (RedirUrl->IsOk(usHttp)){
      //SaveToErrLog((TStr("Redirect: ")+HttpResp->GetHdStr()).CStr());
      TStr Host=HttpResp->GetFldVal(THttp::HostFldNm);
      WebNetCltV->FetchUrl(RedirUrl, FetchId, Host);
    } else {
      TStr MsgStr=TStr("Invalid Redirection URL (")+RedirUrlStr+")";
      WebNetCltV->OnHttpRespError(FetchId, MsgStr);
    }
  } else { // all other codes - error
    TStr MsgStr=TStr("Http Error (")+
     TInt::GetStr(StatusCd)+"/"+HttpResp->GetReasonPhrase()+")";
    WebNetCltV->OnHttpRespError(FetchId, MsgStr);
  }
}
예제 #5
0
void TSAppSrvFun::LogReqRes(const TStrKdV& FldNmValPrV, const PHttpResp& HttpResp)
{
	if (LogRqFolder == "")
		return;
	try	 {
		TDir::GenDir(LogRqFolder);
		TStr TimeNow = TTm::GetCurLocTm().GetWebLogDateTimeStr(true);
		TimeNow.ChangeChAll(':', '.');
		PSOut Output = TFOut::New(LogRqFolder + "/" + TimeNow + ".txt", false);
		Output->PutStr(FunNm.CStr()); Output->PutCh('\n');
		for (int N=0; N < FldNmValPrV.Len(); N++)
			Output->PutStrFmt("  %s: %s\n", FldNmValPrV[N].Key.CStr(), FldNmValPrV[N].Dat.CStr());
		Output->PutCh('\n');
		Output->PutStr(HttpResp->GetBodyAsStr(), false);
	} catch (...) {
		/*const PNotify& Notify = RqEnv->GetWebSrv()->GetNotify();
		Notify->OnStatus("Unable to log request for function '" + GetFunNm() + "'!");*/
	}
}
예제 #6
0
bool IsCTxtHttpResp(const PUrl& Url, const PHttpResp& HttpResp, const int& MnCTxtToks){
  if (HttpResp->IsStatusCd_Ok()){
    PWebPg WebPg=TWebPg::New(Url->GetUrlStr(), HttpResp);
    if (HttpResp->IsContType(THttp::TextHtmlFldVal)){
      TMem BodyMem=HttpResp->GetBodyAsMem();
      PSIn BodyMemIn=TMemIn::New(BodyMem);
      // prepare html-tokens
      PHtmlDoc HtmlDoc=THtmlDoc::New(BodyMemIn, hdtAll, false);
      int Toks=HtmlDoc->GetToks(); THtmlLxSym TokSym; TStr TokStr;
      // prepare continuous-text indicators
      int CTxtToks=0; TChA CTxtChA; bool CTxtP=false;
      // prepare script & style flag
      bool InScript=false; bool InStyle=false; 
      // traverse tokens
      for (int TokN=0; TokN<Toks; TokN++){
        // get token data
        HtmlDoc->GetTok(TokN, TokSym, TokStr);
        switch (TokSym){
          case hsyStr:
          case hsyNum:
          case hsySSym:
            if (!InScript&&!InStyle){
              // text token
              CTxtToks++; CTxtChA+=TokStr; CTxtChA+=' '; 
            }
            break;
          case hsyBTag:
            if (!InScript&&!InStyle){
              if (TokStr=="<SCRIPT>"){
                // start of script
                InScript=true; CTxtToks=0; CTxtChA.Clr();
              } else 
              if (TokStr=="<STYLE>"){
                // start of style
                InStyle=true; CTxtToks=0; CTxtChA.Clr();
              } else {
                if ((TokStr=="<P>")||(TokStr=="<B>")||(TokStr=="<I>")){
                  // skip in-text-tags
                } else {
                  // non-text-tags - break continuous-text
                  CTxtToks=0; CTxtChA.Clr();
                }
              }
            }
            break;
          case hsyETag:
            if (InScript||InStyle){
              if (TokStr=="<SCRIPT>"){
                // end of script
                InScript=false;
              } else
              if (TokStr=="<STYLE>"){
                // end of style
                InStyle=false;
              }
            }
            break;
          default: 
            // non-text-token - break continuous-text
            CTxtToks=0; CTxtChA.Clr();
            break;
        }
        // stop if enough continuous-text
        if (CTxtToks>MnCTxtToks){
          CTxtP=true; break;
        }
      }
      if (CTxtP){
        printf("%s\n", Url->GetUrlStr().CStr());
      }
      return CTxtP;
    }
  }
  return false;
}
void TWebNetSrv::SendHttpResp(const int& SockId, const PHttpResp& HttpResp){
  TMem Mem; HttpResp->GetAsMem(Mem);
  PNetObj NetObj=PNetObj(new TNetMem(Mem));
  SendNetObj(SockId, NetObj);
}
void TMongCliContext::Respond(const PHttpResp& HttpResp) {
	const TMem& Body = HttpResp->GetBodyAsMem();
	TStr ResponseHeader = HttpResp->GetHdStr();
	mg_write(Conn, ResponseHeader.CStr(), ResponseHeader.Len());
	mg_write(Conn, Body.GetBf(), Body.Len());
}