Selection selectNextWord (const Document & document, const Selection & basis) {
// Move until a non-word character.
DocumentIterator cursor = document.at(basis.extent());
while (cursor != document.end() && isWordCharacter(*cursor)) {
++cursor;
}
if (cursor == document.end()) {
return basis;
}
// Then move until a word character.
while (cursor != document.end() && !isWordCharacter(*cursor)) {
++cursor;
}
if (cursor == document.end()) {
return basis;
}
// Save the origin, and move to the next non-word character...
Location origin = cursor.location();
while (cursor != document.end() && isWordCharacter(*cursor)) {
++cursor;
}
// ...but don't actually include that non-word character.
if (cursor != document.end()) {
--cursor;
}
return Selection(origin, cursor.location());
}
Selection selectPriorWord (const Document & document, const Selection & basis) {
// Move until a non-word character.
DocumentIterator cursor = document.at(basis.extent());
while (cursor != document.begin() && isWordCharacter(*cursor)) {
--cursor;
}
// Then move until a word character.
while (cursor != document.begin() && !isWordCharacter(*cursor)) {
--cursor;
}
// Save the extent, and move to the next non-word character...
Location extent = cursor.location();
while (cursor != document.begin() && isWordCharacter(*cursor)) {
--cursor;
}
// ...but don't actually include that non-word character.
if (cursor != document.begin()) {
++cursor;
}
return Selection(cursor.location(), extent);
}
int KateHlWordDetect::checkHgl(const QString &text, int offset, int len)
{
//NOTE: word boundary means: any non-word character.
// make sure there is no letter or number before the word starts
if (offset > 0 && isWordCharacter(text.at(offset - 1))) {
return 0;
}
offset = KateHlStringDetect::checkHgl(text, offset, len);
// make sure there is no letter or number after the word ends
if (offset && offset < text.length() && isWordCharacter(text.at(offset))) {
return 0;
}
return offset;
}
Selection selectThisWord (const Document & document, const Selection & basis) {
// Move backwards until after a space.
DocumentIterator origin = document.at(basis.origin());
while (origin != document.begin() && isWordCharacter(*origin)) {
--origin;
}
++origin;
// Move forwards until before a space.
DocumentIterator extent = document.at(basis.extent());
while (extent != document.end() && isWordCharacter(*extent)) {
++extent;
}
--extent;
return Selection(origin.location(), extent.location());
}
/*
* Implementation notes: scanWord
* ------------------------------
* Reads characters until the scanner reaches the end of a sequence
* of word characters.
*/
std::string TokenScanner::scanWord() {
std::string token = "";
while (true) {
int ch = isp->get();
if (ch == EOF) {
break;
}
if (!isWordCharacter(ch)) {
isp->unget();
break;
}
token += char(ch);
}
return token;
}
TokenType TokenScanner::getTokenType(const std::string& token) const {
if (token.empty()) {
return TokenType(EOF);
}
char ch = token[0];
if (isspace(ch)) {
return SEPARATOR;
} else if (ch == '"' || (ch == '\'' && token.length() > 1)) {
return STRING;
} else if (isdigit(ch)) {
return NUMBER;
} else if (isWordCharacter(ch)) {
return WORD;
} else {
return OPERATOR;
}
}
int main()
{
int ch, size, len;
char* word;
ch = getchar();
while(ch != EOF)
{
if(isWordCharacter(ch))
{
len++; // Adding another character
if(size == 0) // Beginning of new word
{
size = 4;
word = malloc(size * sizeof(char));
}
else if(len >= size) // Need to allocate more memory for word
{
size *= 2;
word = realloc(word, size * sizeof(char));
}
word[len - 1] = ch;
}
else // ch is not a word character
{
if(len > 0) // A word has been prepared
{
word[len] = '\0'; // Add null-terminator
count(word);
// Prepare for next word
//free(word);
//word = NULL;
size = 0;
len = 0;
}
// Else ignore ch
}
ch = getchar();
}
// Finish the last word if necessary
if(len > 0) // A word has been prepared
{
word[len] = '\0'; // Add null-terminator
count(word);
// Prepare for next word
//free(word);
//word = NULL;
//size = 0;
//len = 0;
}
//printf("%d unique words counted:\n", counted.len);
{
int i;
for(i = 0; i < counted.len; i++)
{
/*printf("%d: (\"%s\", %d), hash %ld\n", i + 1, counted.pairs[i]->word,
counted.pairs[i]->count, hash(counted.pairs[i]->word) % HASHBINS);*/
printf("%d: (\"%s\", %d)\n", i + 1, counted.pairs[i]->word,
counted.pairs[i]->count);
// Clean up
free(counted.pairs[i]->word);
free(counted.pairs[i]);
}
}
//printf("A total of %ld string comparisons took place.\n", comparisons);
//printf("%ld words were counted.\n", words);
free(counted.pairs);
return 0;
}
std::string TokenScanner::nextToken() {
if (savedTokens) {
StringCell* cp = savedTokens;
std::string token = cp->str;
savedTokens = cp->link;
delete cp;
return token;
}
while (true) {
if (ignoreWhitespaceFlag) {
skipSpaces();
}
int ch = isp->get();
if (ch == '/' && ignoreCommentsFlag) {
ch = isp->get();
if (ch == '/') {
while (true) {
ch = isp->get();
if (ch == '\n' || ch == '\r' || ch == EOF) {
break;
}
}
continue;
} else if (ch == '*') {
int prev = EOF;
while (true) {
ch = isp->get();
if (ch == EOF || (prev == '*' && ch == '/')) {
break;
}
prev = ch;
}
continue;
}
if (ch != EOF) {
isp->unget();
}
ch = '/';
}
if (ch == EOF) {
return "";
}
if ((ch == '"' || ch == '\'') && scanStringsFlag) {
isp->unget();
return scanString();
}
if (isdigit(ch) && scanNumbersFlag) {
isp->unget();
return scanNumber();
}
if (isWordCharacter(ch)) {
isp->unget();
return scanWord();
}
std::string op = std::string(1, ch);
while (isOperatorPrefix(op)) {
ch = isp->get();
if (ch == EOF) {
break;
}
op += ch;
}
while (op.length() > 1 && !isOperator(op)) {
isp->unget();
op.erase(op.length() - 1, 1);
}
return op;
}
}
int main()
{
char ch;
int size = 0, len = 0;
char* word;
ch = getchar();
while(ch != EOF)
{
if(isWordCharacter(ch))
{
len++; // Adding another character
if(size == 0) // Beginning of new word
{
size = 32;
word = malloc(size * sizeof(char));
}
else if(len >= size) // Need to allocate more memory for word
{
size *= 2;
word = realloc(word, size * sizeof(char));
}
word[len - 1] = ch;
}
else // ch is not a word character
{
if(len > 0) // A word has been prepared
{
word[len] = '\0'; // Add null-terminator
count(word);
len = 0; // Prepare for next word
}
// Else ignore ch
}
ch = getchar();
}
// Finish the last word if necessary
if(len > 0) // A word has been prepared
{
word[len] = '\0'; // Add null-terminator
count(word);
}
free(word);
// Print words
{
int i, j;
for(i = 0; i < HASHBINS; i++)
{
if(table[i])
{
for(j = 0; j < table[i]->len; j++)
{
//printf("(\"%s\", %d)\n", table[i]->pairs[j]->word, table[i]->pairs[j]->count);
// Clean up
free(table[i]->pairs[j]->word);
free(table[i]->pairs[j]);
}
}
free(table[i]);
}
}
printf("A total of %ld string comparisons took place.\n", comparisons);
printf("%ld words were counted.\n", words);
return 0;
}
