void MainWindow::on_selectFile_clicked()
{
QFileDialog dialog;
if (dialog.exec())
{
if (!dialog.selectedFiles().size()) return;
int counter = 0;
QFile file(dialog.selectedFiles()[0]);
file.open(QFile::ReadOnly);
Tokenizer tokenizer(file);
Token tok;
QString sb;
QString sb1k;
QString sb32k;
do
{
tok = tokenizer.Next();
if (tok.GetType() == Token::eof) break;
if (!(counter++ % 256))
{
std::cout << "counter: " << counter << std::endl;
}
sb1k = sb1k % tok.GetText();
if (sb1k.size() == 256)
{
sb32k = sb32k % sb1k;
sb1k.clear();
if (sb32k.size() == 32768)
{
sb = sb % sb32k;
sb32k.clear();
}
}
} while (true);
if (sb1k.size())
{
sb32k = sb32k % sb1k;
}
if (sb32k.size())
{
sb = sb % sb32k;
}
ui->textBrowser->setText(sb);
file.close();
}
}
void TestToken::TestText()
{
SpeechSet* speechSet = new SpeechSet();
Speech* speech = new Speech(speechSet);
Segment* segment = Segment::CreateWithDuration(0, 10000, speech);
Token* token = Token::CreateWithDuration(0, 0, segment);
token->SetSourceText("hop");
assert(token->GetText() == "hop");
//JGF Note: I only added the full test cases for optional frags for the SetProperty("align.optionally", "both") test
Properties::SetProperty("align.optionally", "both");
// (hop)
token->SetSourceText("(hop)");
assert(token->GetSourceText() == "(hop)");
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
assert(token->IsOptional());
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop");
// hop-
Properties::SetProperty("align.fragment_are_correct", "false");
token->SetSourceText("hop-");
assert(token->GetSourceText() == "hop-");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::BEGIN_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop-");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
// (hop-)
Properties::SetProperty("align.fragment_are_correct", "false");
token->SetSourceText("(hop-)");
assert(token->GetSourceText() == "(hop-)");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::BEGIN_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop-");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
// -hop
Properties::SetProperty("align.fragment_are_correct", "false");
token->SetSourceText("-hop");
assert(token->GetSourceText() == "-hop");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::END_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "-hop");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
// (-hop)
Properties::SetProperty("align.fragment_are_correct", "false");
token->SetSourceText("(-hop)");
assert(token->GetSourceText() == "(-hop)");
Properties::SetProperty("align.fragment_are_correct", "true");
cout << "Get " << token->GetText() << endl;
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::END_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "-hop");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.optionally", "ref");
speechSet->SetOrigin("ref");
token->SetSourceText("(hop)");
assert(token->GetSourceText() == "(hop)");
assert(token->GetText() == "hop");
assert(token->IsOptional());
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop");
token->SetSourceText("(hop-)");
assert(token->GetSourceText() == "(hop-)");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::BEGIN_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop-");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.optionally", "ref");
speechSet->SetOrigin("hyp");
token->SetSourceText("(hop)");
assert(token->GetSourceText() == "(hop)");
assert(token->GetText() == "(hop)");
assert(!token->IsOptional());
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "(hop)");
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "(hop)");
token->SetSourceText("(hop-)");
assert(token->GetSourceText() == "(hop-)");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "(hop-)");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "(hop-)");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.optionally", "hyp");
speechSet->SetOrigin("hyp");
token->SetSourceText("(hop)");
assert(token->GetSourceText() == "(hop)");
assert(token->GetText() == "hop");
assert(token->IsOptional());
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop");
token->SetSourceText("(hop-)");
assert(token->GetSourceText() == "(hop-)");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "hop");
assert(token->GetFragmentStatus() == Token::BEGIN_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "hop-");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.optionally", "hyp");
speechSet->SetOrigin("ref");
token->SetSourceText("(hop)");
assert(token->GetSourceText() == "(hop)");
assert(token->GetText() == "(hop)");
assert(!token->IsOptional());
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "(hop)");
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "(hop)");
token->SetSourceText("(hop-)");
assert(token->GetSourceText() == "(hop-)");
Properties::SetProperty("align.fragment_are_correct", "true");
assert(token->GetText() == "(hop-)");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
Properties::SetProperty("align.fragment_are_correct", "false");
assert(token->GetText() == "(hop-)");
assert(token->GetFragmentStatus() == Token::NOT_FRAGMENT);
delete token;
delete segment;
delete speech;
delete speechSet;
}