bool HttpMessage::ParseHeaders(const StringPiece& data, HttpMessage::ErrorCode* error) {
HttpMessage::ErrorCode error_placeholder;
if (error == NULL)
error = &error_placeholder;
StringPiece::size_type pos = data.find_first_of('\n');
if (pos == StringPiece::npos) {
pos = data.size();
}
std::string first_line =
StringTrimRight(data.substr(0, pos), "\r");
if (first_line.empty()) {
*error = HttpMessage::ERROR_NO_START_LINE;
return false;
}
if (!ParseStartLine(first_line, error))
return false;
int error_code = 0;
bool result = m_headers.Parse(data.substr(pos + 1), &error_code);
*error = static_cast<HttpMessage::ErrorCode>(error_code);
return result;
}
static inline
void SplitStringToIteratorUsing(const StringPiece& full, const char* delim, ITR& result)
{
// Optimize the common case where delim is a single character.
if (delim[0] != '\0' && delim[1] == '\0')
{
char c = delim[0];
const char* p = full.data();
const char* end = p + full.size();
while (p != end)
{
if (*p == c)
++p;
else
{
const char* start = p;
while (++p != end && *p != c) {}
*result++ = StringType(start, p - start);
}
}
return;
}
std::string::size_type begin_index, end_index;
begin_index = full.find_first_not_of(delim);
while (begin_index != std::string::npos)
{
end_index = full.find_first_of(delim, begin_index);
if (end_index == std::string::npos)
{
*result++ = full.substr(begin_index).as_string();
return;
}
*result++ = full.substr(begin_index, (end_index - begin_index)).as_string();
begin_index = full.find_first_not_of(delim, end_index);
}
}
void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range, ChartParserCallback &to)
{
// unknown word, add as trans opt
const StaticData &staticData = StaticData::Instance();
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer = UnknownWordPenaltyProducer::Instance();
size_t isDigit = 0;
if (staticData.GetDropUnknown()) {
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
const StringPiece s = f->GetString();
isDigit = s.find_first_of("0123456789");
if (isDigit == string::npos)
isDigit = 0;
else
isDigit = 1;
// modify the starting bitmap
}
Phrase* unksrc = new Phrase(1);
unksrc->AddWord() = sourceWord;
Word &newWord = unksrc->GetWord(0);
newWord.SetIsOOV(true);
m_unksrcs.push_back(unksrc);
//TranslationOption *transOpt;
if (! staticData.GetDropUnknown() || isDigit) {
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
UnknownLHSList::const_iterator iterLHS;
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
const string &targetLHSStr = iterLHS->first;
float prob = iterLHS->second;
// lhs
//const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
Word *targetLHS = new Word(true);
targetLHS->CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
// add to dictionary
TargetPhrase *targetPhrase = new TargetPhrase();
Word &targetWord = targetPhrase->AddWord();
targetWord.CreateUnknownWord(sourceWord);
// scores
float unknownScore = FloorScore(TransformScore(prob));
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
targetPhrase->Evaluate(*unksrc);
targetPhrase->SetTargetLHS(targetLHS);
targetPhrase->SetAlignmentInfo("0-0");
if (staticData.IsDetailedTreeFragmentsTranslationReportingEnabled()) {
targetPhrase->SetProperty("Tree","[ " + (*targetLHS)[0]->GetString().as_string() + " "+sourceWord[0]->GetString().as_string()+" ]");
}
// chart rule
to.AddPhraseOOV(*targetPhrase, m_cacheTargetPhraseCollection, range);
} // for (iterLHS
} else {
// drop source word. create blank trans opt
float unknownScore = FloorScore(-numeric_limits<float>::infinity());
TargetPhrase *targetPhrase = new TargetPhrase();
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
UnknownLHSList::const_iterator iterLHS;
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
const string &targetLHSStr = iterLHS->first;
//float prob = iterLHS->second;
Word *targetLHS = new Word(true);
targetLHS->CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
UTIL_THROW_IF2(targetLHS->GetFactor(0) == NULL, "Null factor for target LHS");
targetPhrase->GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
targetPhrase->Evaluate(*unksrc);
targetPhrase->SetTargetLHS(targetLHS);
// chart rule
to.AddPhraseOOV(*targetPhrase, m_cacheTargetPhraseCollection, range);
}
}
}