vector<Token> TokenizerEn::TokenizeWithAlignment(
const wstring& sentence, vector<size_t>& o_positions) const
{
vector<wstring> splitted;
o_positions.clear();
vector<Token> tokens = processSentence(sentence, splitted, o_positions);
/* for(size_t index = 0; index < tokens.size(); index++) {
std::wcout << tokens[index].content << L" ";
}
std::wcout << endl;
*/
return tokens;
}
void StyledTextFileType::rtfreader(std::istream &ifs,StyledDocumentRefPtr &Result)
{
std::string sentence;
Int32 count = 1;
while(getline(ifs,sentence))
{
if(sentence.find("rtf")!= std::string::npos)
{
continue;
}
if(sentence.find("fonttbl")!= std::string::npos)
{
std::string slashf = "\\f";
Int32 count = -1;
while(1)
{
char countstr[100];
sprintf(countstr,"%d",(count+1));
if(sentence.find(slashf+countstr)!= std::string::npos)
{
Int32 index = sentence.find(slashf+(countstr));
bool nospaceyet= true;
std::string font = "";
for(Int32 i = index;i<sentence.size();i++)
{
if(sentence[i]==';')break;
if(sentence[i]==' ' && nospaceyet){
nospaceyet = false;
continue;
}
if(!nospaceyet)font+=sentence[i];
}
_FontFaces.push_back(font);
count++;
}
else break;
}
Result->setFonts(_FontFaces);
continue;
}
if(sentence.find("colortbl")!= std::string::npos)
{
Color3f _DefaultColor(0.0f,0.0f,0.0f);
_Colors.push_back(_DefaultColor);
Int32 index = findNextSem(0,sentence);
while(index < sentence.size())
{
Color3f col(static_cast<Real32>(findNextRed(index,sentence))/255.0f,
static_cast<Real32>(findNextGreen(index,sentence))/255.0f,
static_cast<Real32>(findNextBlue(index,sentence))/255.0f);
index = findNextSem(index,sentence);
if(col.red()>=0.0f && col.green()>=0.0f && col.blue()>=0.0f)
{
_Colors.push_back(col);
}
}
Result->setColors(_Colors);
continue;
}
if(sentence[0]=='}')
{
break;
}
DocumentElementAttribute _TextStructure;
processSentence(sentence,_TextStructure);
_DocumentElementAttribute.push_back(_TextStructure);
count++;
}
}
