bool CharTokenizer::incrementToken() { clearAttributes(); int32_t length = 0; int32_t start = bufferIndex; CharArray buffer(termAtt->termBuffer()); while (true) { if (bufferIndex >= dataLen) { offset += dataLen; dataLen = input->read(ioBuffer.get(), 0, ioBuffer.size()); if (dataLen == -1) { dataLen = 0; // so next offset += dataLen won't decrement offset if (length > 0) break; else return false; } bufferIndex = 0; } wchar_t c = ioBuffer[bufferIndex++]; if (isTokenChar(c)) // if it's a token char { if (length == 0) start = offset + bufferIndex - 1; else if (length == buffer.size()) buffer = termAtt->resizeTermBuffer(1 + length); buffer[length++] = normalize(c); // buffer it, normalized if (length == MAX_WORD_LEN) // buffer overflow! break; } else if (length > 0) // at non-Letter with chars break; // return them } termAtt->setTermLength(length); offsetAtt->setOffset(correctOffset(start), correctOffset(start + length)); return true; }
bool StandardTokenizer::incrementToken() { clearAttributes(); int32_t posIncr = 1; while (true) { int32_t tokenType = scanner->getNextToken(); if (tokenType == StandardTokenizerImpl::YYEOF) { return false; } if (scanner->yylength() <= maxTokenLength) { posIncrAtt->setPositionIncrement(posIncr); scanner->getText(termAtt); int32_t start = scanner->yychar(); offsetAtt->setOffset(correctOffset(start), correctOffset(start + termAtt->termLength())); // This 'if' should be removed in the next release. For now, it converts invalid acronyms to HOST. /// When removed, only the 'else' part should remain. if (tokenType == ACRONYM_DEP) { if (replaceInvalidAcronym) { typeAtt->setType(TOKEN_TYPES()[HOST]); termAtt->setTermLength(termAtt->termLength() - 1); // remove extra '.' } else { typeAtt->setType(TOKEN_TYPES()[ACRONYM]); } } else { typeAtt->setType(TOKEN_TYPES()[tokenType]); } return true; } else { // When we skip a too-long term, we still increment the position increment ++posIncr; } } }
void StandardTokenizer::end() { // set final offset int32_t finalOffset = correctOffset(scanner->yychar() + scanner->yylength()); offsetAtt->setOffset(finalOffset, finalOffset); }
void CharTokenizer::end() { // set final offset int32_t finalOffset = correctOffset(offset); offsetAtt->setOffset(finalOffset, finalOffset); }
bool CJKTokenizer::incrementToken() { clearAttributes(); while (true) // loop until we find a non-empty token { int32_t length = 0; // the position used to create Token int32_t start = offset; while (true) // loop until we've found a full token { wchar_t c = 0; UnicodeBlock ub = NONE; ++offset; if (bufferIndex >= dataLen) { dataLen = input->read(ioBuffer.get(), 0, ioBuffer.size()); bufferIndex = 0; } if (dataLen == -1) { if (length > 0) { if (preIsTokened == true) { length = 0; preIsTokened = false; } else --offset; break; } else { --offset; return false; } } else { // get current character c = ioBuffer[bufferIndex++]; // get the UnicodeBlock of the current character ub = unicodeBlock(c); } // if the current character is ASCII or Extend ASCII if (ub == BASIC_LATIN || ub == HALFWIDTH_AND_FULLWIDTH_FORMS) { if (ub == HALFWIDTH_AND_FULLWIDTH_FORMS) { int32_t i = (int32_t)c; if (i >= 65281 && i <= 65374) { // convert certain HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN i = i - 65248; c = (wchar_t)i; } } // if the current character is a letter or "_" "+" "#" if (UnicodeUtil::isAlnum(c) || c == L'_' || c == L'+' || c == L'#') { if (length == 0) { // "javaC1C2C3C4linux" <br> // ^--: the current character begin to token the ASCII // letter start = offset - 1; } else if (tokenType == DOUBLE_TOKEN_TYPE) { // "javaC1C2C3C4linux" <br> // ^--: the previous non-ASCII // : the current character --offset; --bufferIndex; if (preIsTokened) { // there is only one non-ASCII has been stored length = 0; preIsTokened = false; break; } else break; } // store the LowerCase(c) in the buffer buffer[length++] = CharFolder::toLower(c); tokenType = SINGLE_TOKEN_TYPE; // break the procedure if buffer overflowed! if (length == MAX_WORD_LEN) break; } else if (length > 0) { if (preIsTokened) { length = 0; preIsTokened = false; } else break; } } else { // non-ASCII letter, e.g."C1C2C3C4" if (UnicodeUtil::isAlpha(c)) { if (length == 0) { start = offset - 1; buffer[length++] = c; tokenType = DOUBLE_TOKEN_TYPE; } else { if (tokenType == SINGLE_TOKEN_TYPE) { --offset; --bufferIndex; // return the previous ASCII characters break; } else { buffer[length++] = c; tokenType = DOUBLE_TOKEN_TYPE; if (length == 2) { --offset; --bufferIndex; preIsTokened = true; break; } } } } else if (length > 0) { if (preIsTokened) { // empty the buffer length = 0; preIsTokened = false; } else break; } } } if (length > 0) { termAtt->setTermBuffer(buffer.get(), 0, length); offsetAtt->setOffset(correctOffset(start), correctOffset(start + length)); typeAtt->setType(TOKEN_TYPE_NAMES[tokenType]); return true; } else if (dataLen == -1) { --offset; return false; } // Cycle back and try for the next token (don't return an empty string) } }
void QFRDRFCSCrossCorrelationEditor::createWidgets() { correlationMaskTools=new QFCorrelationMaskTools(this); connect(correlationMaskTools, SIGNAL(maskChanged()), this, SLOT(rawDataChangedRecalc())); QVBoxLayout* l=new QVBoxLayout(); setLayout(l); splitter=new QVisibleHandleSplitter(Qt::Horizontal, this); l->addWidget(splitter); QWidget* w=new QWidget(this); QGridLayout* gl=new QGridLayout(); w->setLayout(gl); cmbAverageErrors=new QComboBox(w); gl->addWidget(new QLabel(tr("display average:")), 0, 0); gl->addWidget(cmbAverageErrors, 0, 1); cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_enone.png"), tr("no average")); cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_enone.png"), tr("no errors")); cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_elines.png"), tr("with error lines")); cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_ebars.png"), tr("with error bars")); cmbAverageErrors->addItem(QIcon(":/fcs/fcsplot_elinesbars.png"), tr("with lines and bars")); cmbAverageErrors->addItem(QIcon(":/libqf3widgets/plot_epoly.png"), tr("with error polygons")); cmbAverageErrors->addItem(QIcon(":/libqf3widgets/plot_epolybars.png"), tr("with polygons and bars")); connect(cmbAverageErrors, SIGNAL(currentIndexChanged(int)), this, SLOT(replotData(int))); cmbRunDisplay=new QComboBox(w); gl->addWidget(new QLabel(tr("display runs:")), 1, 0); gl->addWidget(cmbRunDisplay, 1, 1); cmbRunDisplay->addItem(tr("average only")); cmbRunDisplay->addItem(tr("selected run")); connect(cmbRunDisplay, SIGNAL(currentIndexChanged(int)), this, SLOT(runsModeChanged(int))); cmbRunErrors=new QComboBox(w); gl->addWidget(new QLabel(tr("run errors:")), 2, 0); gl->addWidget(cmbRunErrors, 2, 1); cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_enone.png"), tr("no errors")); cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_elines.png"), tr("with error lines")); cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_ebars.png"), tr("with error bars")); cmbRunErrors->addItem(QIcon(":/fcs/fcsplot_elinesbars.png"), tr("with lines and bars")); cmbRunErrors->addItem(QIcon(":/libqf3widgets/plot_epoly.png"), tr("with error polygons")); cmbRunErrors->addItem(QIcon(":/libqf3widgets/plot_epolybars.png"), tr("with polygons and bars")); connect(cmbRunErrors, SIGNAL(currentIndexChanged(int)), this, SLOT(replotData(int))); lstRunsSelect=new QListView(w); lstRunsSelect->setModel(runs); lstRunsSelect->setSelectionMode(QAbstractItemView::ExtendedSelection); connect(lstRunsSelect->selectionModel(), SIGNAL(selectionChanged(const QItemSelection &, const QItemSelection &)), this, SLOT(selectionChanged(const QItemSelection &, const QItemSelection &))); connect(lstRunsSelect->selectionModel(), SIGNAL(currentChanged(const QModelIndex &, const QModelIndex &)), this, SLOT(selectionChanged(const QModelIndex &, const QModelIndex &))); gl->addWidget(new QLabel(tr("select runs to display:")), 3, 0, 1, 2); gl->addWidget(lstRunsSelect, 4, 0, 3, 2); gl->setRowStretch(4, 2); btnDontUse=new QPushButton(tr("exclude selected"), w); connect(btnDontUse, SIGNAL(clicked()), this, SLOT(excludeRuns())); gl->addWidget(btnDontUse, 7, 0, 1, 2); btnUse=new QPushButton(tr("include selected"), w); connect(btnUse, SIGNAL(clicked()), this, SLOT(includeRuns())); gl->addWidget(btnUse, 8, 0, 1, 2); chkLogTauAxis=new QCheckBox("", w); gl->addWidget(new QLabel(tr("log-scale on<br>lag time axis:")), 9, 0); gl->addWidget(chkLogTauAxis, 9, 1); connect(chkLogTauAxis, SIGNAL(toggled(bool)), this, SLOT(replotData())); grpInfo=new QGroupBox(tr("Info"), w); QGridLayout* ggl=new QGridLayout(); grpInfo->setLayout(ggl); ggl->addWidget(new QLabel(tr("# Runs = ")), 0, 0); labRuns=new QLabel(grpInfo); ggl->addWidget(labRuns, 0,1); ggl->addWidget(new QLabel(tr("# Points = ")), 1, 0); labCorrelationPoints=new QLabel(grpInfo); ggl->addWidget(labCorrelationPoints, 1,1); gl->addWidget(grpInfo, 10,0,1,2); grpRelCCF=new QGroupBox(tr("rel. CCF"), w); QFormLayout* ggrcl=new QFormLayout(); grpRelCCF->setLayout(ggrcl); cmbFCCSCorrected=new QComboBox(this); cmbFCCSCorrected->addItem(tr("corrected amplitudes")); cmbFCCSCorrected->addItem(tr("explained amplitudes")); ggrcl->addRow(tr("rel. CCF display mode: "), cmbFCCSCorrected); spinCCFAmplitudeRangeMin=new QDoubleSpinBox(this); spinCCFAmplitudeRangeMin->setRange(0,1000000); spinCCFAmplitudeRangeMin->setValue(5); spinCCFAmplitudeRangeMin->setDecimals(1); spinCCFAmplitudeRangeMin->setSuffix(" microseconds"); ggrcl->addRow(tr("amplitude range, min: "), spinCCFAmplitudeRangeMin); spinCCFAmplitudeRangeMax=new QDoubleSpinBox(this); spinCCFAmplitudeRangeMax->setRange(0,1000000); spinCCFAmplitudeRangeMax->setValue(30); spinCCFAmplitudeRangeMax->setDecimals(1); spinCCFAmplitudeRangeMax->setSuffix(" microseconds"); ggrcl->addRow(tr("amplitude range, max: "), spinCCFAmplitudeRangeMax); spinCrosstalk=new QDoubleSpinBox(this); spinCrosstalk->setRange(0,1000); spinCrosstalk->setValue(0); spinCrosstalk->setDecimals(2); spinCrosstalk->setSuffix(" %"); ggrcl->addRow(tr("G->R crosstalk: "), spinCrosstalk); labRelCCF=new QLabel(grpRelCCF); ggrcl->addRow(tr("Results, average: "), labRelCCF); labRelCCFSel=new QLabel(grpRelCCF); ggrcl->addRow(labRelCCFSelLab=new QLabel(tr("Results, run ?: ")), labRelCCFSel); gl->addWidget(grpRelCCF, 11,0,1,2); QWidget* wp=new QWidget(this); QVBoxLayout* lp=new QVBoxLayout(); wp->setLayout(lp); plotter = new QFPlotter(true, this); plotter->get_plotter()->set_userSettigsFilename(ProgramOptions::getInstance()->getIniFilename()); lp->addWidget(plotter); sliders=new DataCutSliders(this); connect(sliders, SIGNAL(slidersChanged(int , int , int, int)), this, SLOT(slidersChanged(int, int, int, int))); lp->addWidget(sliders); splitter->addWidget(wp); splitter->addWidget(w); splitter->setCollapsible(0, false); splitter->setCollapsible(1, false); splitter->setStretchFactor(0,5); splitter->setStretchFactor(1,1); actCopyNormalizedACF=new QFActionWithNoMenuRole(tr("copy normalized CFs to table"), this); connect(actCopyNormalizedACF, SIGNAL(triggered()), this, SLOT(copyNormalizedACFs())); actCorrectOffset=new QFActionWithNoMenuRole(tr("correct CFs for offset"), this); connect(actCorrectOffset, SIGNAL(triggered()), this, SLOT(correctOffset())); menuMask=propertyEditor->addMenu("&Selection/Mask", 0); correlationMaskTools->registerMaskToolsToMenu(menuMask); menuMask->addSeparator(); correlationMaskTools->registerCorrelationToolsToMenu(menuMask); menuData=propertyEditor->addOrFindMenu(tr("&Data"), -1); //menuData->addAction(actCopyNormalizedACF); menuData->addAction(actCorrectOffset); }