bool CharTokenizer::incrementToken()
{
clearAttributes();
int32_t length = 0;
int32_t start = bufferIndex;
CharArray buffer(termAtt->termBuffer());
while (true)
{
if (bufferIndex >= dataLen)
{
offset += dataLen;
dataLen = input->read(ioBuffer.get(), 0, ioBuffer.size());
if (dataLen == -1)
{
dataLen = 0; // so next offset += dataLen won't decrement offset
if (length > 0)
break;
else
return false;
}
bufferIndex = 0;
}
wchar_t c = ioBuffer[bufferIndex++];
if (isTokenChar(c)) // if it's a token char
{
if (length == 0)
start = offset + bufferIndex - 1;
else if (length == buffer.size())
buffer = termAtt->resizeTermBuffer(1 + length);
buffer[length++] = normalize(c); // buffer it, normalized
if (length == MAX_WORD_LEN) // buffer overflow!
break;
}
else if (length > 0) // at non-Letter with chars
break; // return them
}
termAtt->setTermLength(length);
offsetAtt->setOffset(correctOffset(start), correctOffset(start + length));
return true;
}
bool StandardTokenizer::incrementToken() {
clearAttributes();
int32_t posIncr = 1;
while (true) {
int32_t tokenType = scanner->getNextToken();
if (tokenType == StandardTokenizerImpl::YYEOF) {
return false;
}
if (scanner->yylength() <= maxTokenLength) {
posIncrAtt->setPositionIncrement(posIncr);
scanner->getText(termAtt);
int32_t start = scanner->yychar();
offsetAtt->setOffset(correctOffset(start), correctOffset(start + termAtt->termLength()));
// This 'if' should be removed in the next release. For now, it converts invalid acronyms to HOST.
/// When removed, only the 'else' part should remain.
if (tokenType == ACRONYM_DEP) {
if (replaceInvalidAcronym) {
typeAtt->setType(TOKEN_TYPES()[HOST]);
termAtt->setTermLength(termAtt->termLength() - 1); // remove extra '.'
} else {
typeAtt->setType(TOKEN_TYPES()[ACRONYM]);
}
} else {
typeAtt->setType(TOKEN_TYPES()[tokenType]);
}
return true;
} else {
// When we skip a too-long term, we still increment the position increment
++posIncr;
}
}
}
void StandardTokenizer::end() {
// set final offset
int32_t finalOffset = correctOffset(scanner->yychar() + scanner->yylength());
offsetAtt->setOffset(finalOffset, finalOffset);
}
void CharTokenizer::end()
{
// set final offset
int32_t finalOffset = correctOffset(offset);
offsetAtt->setOffset(finalOffset, finalOffset);
}
bool CJKTokenizer::incrementToken()
{
clearAttributes();
while (true) // loop until we find a non-empty token
{
int32_t length = 0;
// the position used to create Token
int32_t start = offset;
while (true) // loop until we've found a full token
{
wchar_t c = 0;
UnicodeBlock ub = NONE;
++offset;
if (bufferIndex >= dataLen)
{
dataLen = input->read(ioBuffer.get(), 0, ioBuffer.size());
bufferIndex = 0;
}
if (dataLen == -1)
{
if (length > 0)
{
if (preIsTokened == true)
{
length = 0;
preIsTokened = false;
}
else
--offset;
break;
}
else
{
--offset;
return false;
}
}
else
{
// get current character
c = ioBuffer[bufferIndex++];
// get the UnicodeBlock of the current character
ub = unicodeBlock(c);
}
// if the current character is ASCII or Extend ASCII
if (ub == BASIC_LATIN || ub == HALFWIDTH_AND_FULLWIDTH_FORMS)
{
if (ub == HALFWIDTH_AND_FULLWIDTH_FORMS)
{
int32_t i = (int32_t)c;
if (i >= 65281 && i <= 65374)
{
// convert certain HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN
i = i - 65248;
c = (wchar_t)i;
}
}
// if the current character is a letter or "_" "+" "#"
if (UnicodeUtil::isAlnum(c) || c == L'_' || c == L'+' || c == L'#')
{
if (length == 0)
{
// "javaC1C2C3C4linux" <br>
// ^--: the current character begin to token the ASCII
// letter
start = offset - 1;
}
else if (tokenType == DOUBLE_TOKEN_TYPE)
{
// "javaC1C2C3C4linux" <br>
// ^--: the previous non-ASCII
// : the current character
--offset;
--bufferIndex;
if (preIsTokened)
{
// there is only one non-ASCII has been stored
length = 0;
preIsTokened = false;
break;
}
else
break;
}
// store the LowerCase(c) in the buffer
buffer[length++] = CharFolder::toLower(c);
tokenType = SINGLE_TOKEN_TYPE;
// break the procedure if buffer overflowed!
if (length == MAX_WORD_LEN)
break;
}
else if (length > 0)
{
if (preIsTokened)
{
length = 0;
preIsTokened = false;
}
else
break;
}
}
else
{
// non-ASCII letter, e.g."C1C2C3C4"
if (UnicodeUtil::isAlpha(c))
{
if (length == 0)
{
start = offset - 1;
buffer[length++] = c;
tokenType = DOUBLE_TOKEN_TYPE;
}
else
{
if (tokenType == SINGLE_TOKEN_TYPE)
{
--offset;
--bufferIndex;
// return the previous ASCII characters
break;
}
else
{
buffer[length++] = c;
tokenType = DOUBLE_TOKEN_TYPE;
if (length == 2)
{
--offset;
--bufferIndex;
preIsTokened = true;
break;
}
}
}
}
else if (length > 0)
{
if (preIsTokened)
{
// empty the buffer
length = 0;
preIsTokened = false;
}
else
break;
}
}
}
if (length > 0)
{
termAtt->setTermBuffer(buffer.get(), 0, length);
offsetAtt->setOffset(correctOffset(start), correctOffset(start + length));
typeAtt->setType(TOKEN_TYPE_NAMES[tokenType]);
return true;
}
else if (dataLen == -1)
{
--offset;
return false;
}
// Cycle back and try for the next token (don't return an empty string)
}
}
void QFRDRFCSCrossCorrelationEditor::createWidgets() {
correlationMaskTools=new QFCorrelationMaskTools(this);
connect(correlationMaskTools, SIGNAL(maskChanged()), this, SLOT(rawDataChangedRecalc()));
QVBoxLayout* l=new QVBoxLayout();
setLayout(l);
splitter=new QVisibleHandleSplitter(Qt::Horizontal, this);
l->addWidget(splitter);
QWidget* w=new QWidget(this);
QGridLayout* gl=new QGridLayout();
w->setLayout(gl);
cmbAverageErrors=new QComboBox(w);
gl->addWidget(new QLabel(tr("display average:")), 0, 0);
gl->addWidget(cmbAverageErrors, 0, 1);
cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_enone.png"), tr("no average"));
cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_enone.png"), tr("no errors"));
cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_elines.png"), tr("with error lines"));
cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_ebars.png"), tr("with error bars"));
cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_elinesbars.png"), tr("with lines and bars"));
cmbAverageErrors->addItem(QIcon(":/libqf3widgets/plot_epoly.png"), tr("with error polygons"));
cmbAverageErrors->addItem(QIcon(":/libqf3widgets/plot_epolybars.png"), tr("with polygons and bars"));
connect(cmbAverageErrors, SIGNAL(currentIndexChanged(int)), this, SLOT(replotData(int)));
cmbRunDisplay=new QComboBox(w);
gl->addWidget(new QLabel(tr("display runs:")), 1, 0);
gl->addWidget(cmbRunDisplay, 1, 1);
cmbRunDisplay->addItem(tr("average only"));
cmbRunDisplay->addItem(tr("selected run"));
connect(cmbRunDisplay, SIGNAL(currentIndexChanged(int)), this, SLOT(runsModeChanged(int)));
cmbRunErrors=new QComboBox(w);
gl->addWidget(new QLabel(tr("run errors:")), 2, 0);
gl->addWidget(cmbRunErrors, 2, 1);
cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_enone.png"), tr("no errors"));
cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_elines.png"), tr("with error lines"));
cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_ebars.png"), tr("with error bars"));
cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_elinesbars.png"), tr("with lines and bars"));
cmbRunErrors->addItem(QIcon(":/libqf3widgets/plot_epoly.png"), tr("with error polygons"));
cmbRunErrors->addItem(QIcon(":/libqf3widgets/plot_epolybars.png"), tr("with polygons and bars"));
connect(cmbRunErrors, SIGNAL(currentIndexChanged(int)), this, SLOT(replotData(int)));
lstRunsSelect=new QListView(w);
lstRunsSelect->setModel(runs);
lstRunsSelect->setSelectionMode(QAbstractItemView::ExtendedSelection);
connect(lstRunsSelect->selectionModel(), SIGNAL(selectionChanged(const QItemSelection &, const QItemSelection &)), this, SLOT(selectionChanged(const QItemSelection &, const QItemSelection &)));
connect(lstRunsSelect->selectionModel(), SIGNAL(currentChanged(const QModelIndex &, const QModelIndex &)), this, SLOT(selectionChanged(const QModelIndex &, const QModelIndex &)));
gl->addWidget(new QLabel(tr("select runs to display:")), 3, 0, 1, 2);
gl->addWidget(lstRunsSelect, 4, 0, 3, 2);
gl->setRowStretch(4, 2);
btnDontUse=new QPushButton(tr("exclude selected"), w);
connect(btnDontUse, SIGNAL(clicked()), this, SLOT(excludeRuns()));
gl->addWidget(btnDontUse, 7, 0, 1, 2);
btnUse=new QPushButton(tr("include selected"), w);
connect(btnUse, SIGNAL(clicked()), this, SLOT(includeRuns()));
gl->addWidget(btnUse, 8, 0, 1, 2);
chkLogTauAxis=new QCheckBox("", w);
gl->addWidget(new QLabel(tr("log-scale on<br>lag time axis:")), 9, 0);
gl->addWidget(chkLogTauAxis, 9, 1);
connect(chkLogTauAxis, SIGNAL(toggled(bool)), this, SLOT(replotData()));
grpInfo=new QGroupBox(tr("Info"), w);
QGridLayout* ggl=new QGridLayout();
grpInfo->setLayout(ggl);
ggl->addWidget(new QLabel(tr("# Runs = ")), 0, 0);
labRuns=new QLabel(grpInfo);
ggl->addWidget(labRuns, 0,1);
ggl->addWidget(new QLabel(tr("# Points = ")), 1, 0);
labCorrelationPoints=new QLabel(grpInfo);
ggl->addWidget(labCorrelationPoints, 1,1);
gl->addWidget(grpInfo, 10,0,1,2);
grpRelCCF=new QGroupBox(tr("rel. CCF"), w);
QFormLayout* ggrcl=new QFormLayout();
grpRelCCF->setLayout(ggrcl);
cmbFCCSCorrected=new QComboBox(this);
cmbFCCSCorrected->addItem(tr("corrected amplitudes"));
cmbFCCSCorrected->addItem(tr("explained amplitudes"));
ggrcl->addRow(tr("rel. CCF display mode: "), cmbFCCSCorrected);
spinCCFAmplitudeRangeMin=new QDoubleSpinBox(this);
spinCCFAmplitudeRangeMin->setRange(0,1000000);
spinCCFAmplitudeRangeMin->setValue(5);
spinCCFAmplitudeRangeMin->setDecimals(1);
spinCCFAmplitudeRangeMin->setSuffix(" microseconds");
ggrcl->addRow(tr("amplitude range, min: "), spinCCFAmplitudeRangeMin);
spinCCFAmplitudeRangeMax=new QDoubleSpinBox(this);
spinCCFAmplitudeRangeMax->setRange(0,1000000);
spinCCFAmplitudeRangeMax->setValue(30);
spinCCFAmplitudeRangeMax->setDecimals(1);
spinCCFAmplitudeRangeMax->setSuffix(" microseconds");
ggrcl->addRow(tr("amplitude range, max: "), spinCCFAmplitudeRangeMax);
spinCrosstalk=new QDoubleSpinBox(this);
spinCrosstalk->setRange(0,1000);
spinCrosstalk->setValue(0);
spinCrosstalk->setDecimals(2);
spinCrosstalk->setSuffix(" %");
ggrcl->addRow(tr("G->R crosstalk: "), spinCrosstalk);
labRelCCF=new QLabel(grpRelCCF);
ggrcl->addRow(tr("Results, average: "), labRelCCF);
labRelCCFSel=new QLabel(grpRelCCF);
ggrcl->addRow(labRelCCFSelLab=new QLabel(tr("Results, run ?: ")), labRelCCFSel);
gl->addWidget(grpRelCCF, 11,0,1,2);
QWidget* wp=new QWidget(this);
QVBoxLayout* lp=new QVBoxLayout();
wp->setLayout(lp);
plotter = new QFPlotter(true, this);
plotter->get_plotter()->set_userSettigsFilename(ProgramOptions::getInstance()->getIniFilename());
lp->addWidget(plotter);
sliders=new DataCutSliders(this);
connect(sliders, SIGNAL(slidersChanged(int , int , int, int)), this, SLOT(slidersChanged(int, int, int, int)));
lp->addWidget(sliders);
splitter->addWidget(wp);
splitter->addWidget(w);
splitter->setCollapsible(0, false);
splitter->setCollapsible(1, false);
splitter->setStretchFactor(0,5);
splitter->setStretchFactor(1,1);
actCopyNormalizedACF=new QFActionWithNoMenuRole(tr("copy normalized CFs to table"), this);
connect(actCopyNormalizedACF, SIGNAL(triggered()), this, SLOT(copyNormalizedACFs()));
actCorrectOffset=new QFActionWithNoMenuRole(tr("correct CFs for offset"), this);
connect(actCorrectOffset, SIGNAL(triggered()), this, SLOT(correctOffset()));
menuMask=propertyEditor->addMenu("&Selection/Mask", 0);
correlationMaskTools->registerMaskToolsToMenu(menuMask);
menuMask->addSeparator();
correlationMaskTools->registerCorrelationToolsToMenu(menuMask);
menuData=propertyEditor->addOrFindMenu(tr("&Data"), -1);
//menuData->addAction(actCopyNormalizedACF);
menuData->addAction(actCorrectOffset);
}
