Exemple #1
0
void CIndex::PrepareKeywords(CSearchObject& SearchObject) const {
    
    Trace(tagSearch, levVerbose, ("CIndex::PrepareKeywords - entering."));
    
    CVector<CString> TmpVector;
    int v, i;
    
    static const CString __SORT("SORT:");
    static const CString __HOST("HOST:");
    static const CString __PATH("PATH:");
    static const CString __URL("URL:");
    static const CString __EXT("EXT:");
    static const CString __OPT("OPT:");
    static const CString __CASE("CASE");
    static const CString __INSENS("INSENS");
    static const CString __BEFORE("BEFORE:");
    static const CString __AFTER("AFTER:");
    static const CString __QUANT("QUANT:");
    static const CString __META("META:");
    
    SearchObject.m_SearchData.m_SearchCaseType = sct_CaseAutomatic;
    SearchObject.m_ParsedSearchString.Empty();
    
    CString MidString;
    
    for (i=(int) SearchObject.m_SearchData.m_Words.GetSize()-1;i>=0;i--){
        if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__SORT)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__SORT.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &SearchObject.m_SortType);
            SearchObject.m_SearchData.m_Words[i].Empty();
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__HOST)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__HOST.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            CString::StrToVector(MidString, ',', &TmpVector);
			for (v=0;v<(int) TmpVector.GetSize();v++) if (TmpVector[v].GetLength()) SearchObject.m_SearchOptions.m_OptionHosts += CUrl::UnEscape(TmpVector[v]);
            SearchObject.m_SearchData.m_Words[i].Empty();
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__PATH)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__PATH.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            CString::StrToVector(MidString, ',', &TmpVector);
            for (v=0;v<(int) TmpVector.GetSize();v++) if (TmpVector[v].GetLength()) SearchObject.m_SearchOptions.m_OptionPaths += CUrl::UnEscape(TmpVector[v]);
            SearchObject.m_SearchData.m_Words[i].Empty();
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__URL)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__URL.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            CString::StrToVector(MidString, ',', &TmpVector);
            for (v=0;v<(int) TmpVector.GetSize();v++) 
				if (TmpVector[v].GetLength()) {
					if (TmpVector[v].Pos("://") != -1) {
						SearchObject.m_SearchOptions.m_OptionUrls += CUrl::UnEscape(TmpVector[v]);
					} else {
						SearchObject.m_SearchOptions.m_OptionUrls += ("http://" + CUrl::UnEscape(TmpVector[v]));
					}
				}
            SearchObject.m_SearchData.m_Words[i].Empty();
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__EXT)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__EXT.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            CString::StrToVector(MidString, ',', &TmpVector);
            for (v=0;v<(int) TmpVector.GetSize();v++) if (TmpVector[v].GetLength()) {
                CString Tmp("."); Tmp += TmpVector[v];
                SearchObject.m_SearchOptions.m_OptionExts.AddSortedUnique(Tmp);
            }
            SearchObject.m_SearchData.m_Words[i].Empty();
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__OPT)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__OPT.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            CString::StrToVector(MidString, ',', &TmpVector);
            for (v=0;v<(int) TmpVector.GetSize();v++) if (TmpVector[v].GetLength()) {
                if (TmpVector[v].Same(__CASE)) {
                    SearchObject.m_SearchData.m_SearchCaseType = sct_CaseSensitive;
                } else if (TmpVector[v].Same(__INSENS)) {
                    SearchObject.m_SearchData.m_SearchCaseType = sct_CaseInsensitive;
                } else SearchObject.m_SearchOptions.m_OptionExtra += TmpVector[v];
            }
            SearchObject.m_SearchData.m_Words[i].Empty();
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__BEFORE)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__BEFORE.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            SearchObject.m_SearchOptions.m_DateBeforeValid = CDate::EncodeSimpleDate(MidString, SearchObject.m_SearchOptions.m_DateBefore, SearchObject.m_SearchOptions.m_USFormats);
            SearchObject.m_SearchData.m_Words.RemoveAt(i);
			continue;
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__AFTER)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__AFTER.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            SearchObject.m_SearchOptions.m_DateAfterValid = CDate::EncodeSimpleDate(MidString, SearchObject.m_SearchOptions.m_DateAfter, SearchObject.m_SearchOptions.m_USFormats);
            SearchObject.m_SearchData.m_Words.RemoveAt(i);
			continue;
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__META)) {
            SearchObject.m_SearchData.m_Words[i].Mid(__META.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            CString::StrToVector(MidString, ',', &TmpVector);
            for (v=0;v<(int) TmpVector.GetSize();v++) 
                if (TmpVector[v].GetLength()) {
                    TmpVector[v].UpperCase();
                    SearchObject.m_SearchOptions.m_OptionMeta += TmpVector[v];
                }
            SearchObject.m_SearchData.m_Words.RemoveAt(i);
			continue;
        } else if (SearchObject.m_SearchData.m_Words[i].StartsWithSame(__QUANT)) {
            int QuantTmp;
            SearchObject.m_SearchData.m_Words[i].Mid(__QUANT.GetLength(), SearchObject.m_SearchData.m_Words[i].GetLength(), &MidString);
            if (MidString.IsInt(&QuantTmp)) {
                if (QuantTmp >= -1) SearchObject.m_Quant = QuantTmp;
            }
        } else {
            SearchObject.m_SearchData.m_Words[i].Trim32();

            if (SearchObject.m_ParsedSearchString.GetLength())
                SearchObject.m_ParsedSearchString.Insert(0, ' ');
            SearchObject.m_ParsedSearchString.Insert(0, SearchObject.m_SearchData.m_Words[i]);
        }

        if (! SearchObject.m_SearchData.m_Words[i].GetLength()) {
            SearchObject.m_SearchData.m_Words.RemoveAt(i);
        } else {
            CAlkalineParser::RemoveAccents(SearchObject.m_SearchData.m_Words[i], m_IndexOptions.m_FreeAlpha);
        }
    }

    // remove accents
	for (i = 0; i < (int) SearchObject.m_Adjustments.GetSize(); i++) {
        CAlkalineParser::RemoveAccents(
			SearchObject.m_Adjustments[i], 
			m_IndexOptions.m_FreeAlpha);
	}
    
    // adjust extra keyword options
    AdjustMetas(SearchObject);
    AdjustKeywords(SearchObject);    

    SearchObject.m_SearchData.m_Words += SearchObject.m_Adjustments;    
    SearchObject.m_SearchData.m_Words.QuickSortUnique();

	// digest search terms, those are lowercase query terms without quotes or markup for highlighting
	// meta tags, negations and alike are ignored
	CString::StrToVector(SearchObject.m_ParsedSearchString, ' ', & SearchObject.m_DigestedSearchTerms);
	for (i = 0; i < (int) SearchObject.m_DigestedSearchTerms.GetSize(); i++) {
		SearchObject.m_DigestedSearchTerms[i].Trim('+');
		SearchObject.m_DigestedSearchTerms[i].Trim('\"');
	}
    
    Trace(tagSearch, levVerbose, ("CIndex::PrepareKeywords - leaving."));
}
Exemple #2
0
bool CDate::EncodeDate(
    const CString& DateString, 
    const CString& DateFormat, 
    struct tm& Result) {

  CString Date(DateString);
  
  static const char * Months[] = {"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"};
  static const char * WeekDays[] = {"Sun", "Mon","Tue","Wed","Thu","Fri","Sat"};
  static const char * Zones[] = {"UTC", "GMT", "EDT", "EST", "CDT", "CST", "MDT", "MST", "PDT", "PST"};
  static const int ZonesHoursSkew[] = {0, 0, 4, 5, 5, 6, 6, 7, 7, 8};
	
  CString WeekDay;
  CString Month;
  CString Zone;
  
  Date.Replace('-', ' ');

  memset(&Result, 0, sizeof(Result));

  bool fScan = 
      SSScan(
          Date, 
          &DateFormat, 
          &WeekDay, 
          NULL, 
          &Result.tm_mday, 
          &Month, 
          &Result.tm_year, 
          &Result.tm_hour, 
          NULL, 
          &Result.tm_min, 
          NULL, 
          &Result.tm_sec, 
          &Zone);
  
  Trace(tagDateTime, levVerbose, 
        ("CDate:: EncodeDate - scanned %s with " \
         "WeekDay: %s " \
         "Day: %d " \
         "Month: %s " \
         "Year: %d " \
         "Hour: %d " \
         "Minute: %d " \
         "Second: %d " \
         "Zone: %s", 
         Date.GetBuffer(),
         WeekDay.GetBuffer(),
         Result.tm_mday,
         Month.GetBuffer(),
         Result.tm_year,
         Result.tm_hour,
         Result.tm_min,
         Result.tm_sec,
         Zone.GetBuffer()));
  
  if (fScan) {

		int i;
      
		if (Result.tm_year > 1900) 
			Result.tm_year-=1900;
    
		for (i = 0; i < BASE_DIM(Months); i++) {
			if (Month.Same(Months[i])) {
				Result.tm_mon = i;
				break;
			}
		}

		for (i = 0; i < BASE_DIM(WeekDays); i++) {
			// support full weekdays as well
			if (WeekDay.StartsWithSame(WeekDays[i])) {
				Result.tm_wday = i;
				break;
			}
		}

		// time zone adjustments
		int ZoneAdjustment = 0;

		if (Zone.IsInt( & ZoneAdjustment)) {
			ZoneAdjustment *= 3600;
		} else {
			for (i = 0; i < BASE_DIM(Zones); i++) {
				if (Zone.Same(Zones[i])) {
					ZoneAdjustment = ZonesHoursSkew[i] * 3600;
					break;
				}
			}
		}

		Trace(tagDateTime, levVerbose, ("CDate:: EncodeSimpleDate - ZoneAdjustment %d second(s)", ZoneAdjustment));

		Result.tm_isdst = 0;

		if (ZoneAdjustment) {
			time_t AdjustedTime;
#ifdef _UNIX
			base_gmtime(AdjustedTime, Result);
#endif
#ifdef _WIN32
			_tzset();
			AdjustedTime = mktime(&Result);
			AdjustedTime -= _timezone;
#endif
			AdjustedTime -= ZoneAdjustment;
			base_localtime(AdjustedTime, Result);
		}

		// Result.tm_gmtoff = ZoneAdjustment;    
		return true;
  }
  return false;
}
Exemple #3
0
bool CHttpRequest::GetHTTP(float HTTPVersion) {   
    
    /* HTTP request */
    CString Request;
    MakeRequestString(HTTPVersion, &Request);

    Trace(tagHttp, levInfo, ("CHttpRequest - Request {\n%s}", Request.GetBuffer()));
    
    if (m_Dump) {
        cout << "-=-=-=-=-=-=-=-=- Request -=-=-=-=-=-=-=-=-" << endl 
             << Request
             << "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-" << endl;
    }
    
    if (!m_ClientSocket.Write(Request))
        return false;
    
    /* HTTP response, header */
    m_ClientSocket.ReadLine(&m_RStatusString);

    Trace(tagHttp, levInfo, ("CHttpRequest - StatusString {%s}", m_RStatusString.GetBuffer()));

    if (! ParseStatusString()) {
        m_RData = m_RStatusString;
    }

    if (m_RStatusValue > 0) {
        // finish reading headers
        CString OneLine;
        m_ClientSocket.ReadLine(&OneLine);

        if (m_Dump) {
            cout << "-=-=-=-=-=-=-=-=- Response -=-=-=-=-=-=-=-" << endl 
                 << m_RStatusString
                 << endl;
        }
        
        Trace(tagHttp, levInfo, ("CHttpRequest - Response (start)    : %s", m_RStatusString.GetBuffer()));
        
        while (OneLine.GetLength()) {
    
            if (m_Dump) {
                cout << OneLine << endl;
            }
            
            CStringTable Table;
            CString Name, Value;            
            if (CMimeParser::ParseLine(OneLine, Name, Value, Table)) {
                Trace(tagHttp, levInfo, ("CHttpRequest - Response (continued): %d | %s <table> (%d)", OneLine.GetLength(), Name.GetBuffer(), Table.GetSize()));
                Trace(tagHttp, levInfo, ("CHttpRequest -                       %s", OneLine.GetBuffer()));
                m_RFields.Add(Name, Table);
                m_RPairFields.Add(CStringPair(Name, Value));
            } else {
                Trace(tagHttp, levInfo, ("CHttpRequest - Response (continued): %s (parser failed, ignoring)", OneLine.GetBuffer()));
                // ignore malformed lines
                // $(TODO): support P3P, which looks like P3P CP= ...
                // m_RStatusValue = 400
                // return false;                
            }
            m_ClientSocket.ReadLine(&OneLine);
        }
    
        if (m_Dump) {
            cout << "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-" << endl;
        }
        
        Trace(tagHttp, levInfo, ("CHttpRequest - Response (done)"));
    }
    
    if ((m_RequestMethod != htGet)&&(m_RequestMethod != htPost)) 
        return true;

    CString ContentLengthString = m_RFields.FindElement(g_strHttpContentLength).GetValue(g_strHttpContentLength);
    
    int ContentLengthExpected = 0;
    
    if (! ContentLengthString.IsInt(& ContentLengthExpected))
        ContentLengthExpected = -1;
    
    Trace(tagHttp, levInfo, ("CHttpRequest::GetHTTP() - expected content-length: %s {%d}", ContentLengthString.GetBuffer(), ContentLengthExpected));        

    m_RData.Empty();
    
    // document is going to be larger than the current maximum document size limit
    if ((ContentLengthExpected > 0) && (m_RequestSizeLimit > 0) && (m_RequestSizeLimit < ContentLengthExpected)) {
        
        m_RStatusValue = 206; // partial content        
        
    } else {
        
        if (ContentLengthExpected > 0) {
    
            m_ClientSocket.Read(& m_RData, ContentLengthExpected);
            
            if (ContentLengthExpected != (int) m_RData.GetLength()) {
	      Trace(tagHttp, levInfo, ("CHttpRequest::GetHTTP() - expected %d bytes, read %d. (207 Partial Content)", ContentLengthExpected, m_RData.GetLength()));        
	      m_RStatusValue = 207; // partial Content-Length
	      return false;
            }
            
        } else if ((ContentLengthExpected < 0) && (m_RStatusValue == 200)) {
            // content-length was not explicitly zero and the status value says that data should be available
            m_ClientSocket.Read(& m_RData, m_RequestSizeLimit);        
        }
        
        if (m_RStatusValue <= 0) {
            m_RStatusValue = (m_RData.GetLength() > 0) ? 200 : 204;
            
            if (!m_RData.GetLength())
                return false;
        }
    }
    
    Trace(tagHttp, levInfo, ("CHttpRequest::GetHTTP() - m_RStatusValue {%d}", m_RStatusValue));        
    Trace(tagHttp, levInfo, ("CHttpRequest::GetHTTP() - m_RData.Length {%d}", m_RData.GetLength()));        
    
    return true;
}