示例#1
0
////////////////////////////////////////////////////////////////////////////////
// Lexer::Type::dom
//   [ <isUUID> | <isDigit>+ . ] <isIdentifier> [ . <isIdentifier> ]*
bool Lexer::isDOM (std::string& token, Lexer::Type& type)
{
  std::size_t marker = _cursor;

  std::string extractedToken;
  Lexer::Type extractedType;
  if (isUUID (extractedToken, extractedType))
  {
    if (_text[_cursor] == '.')
      ++_cursor;
    else
    {
      _cursor = marker;
      return false;
    }
  }
  else
  {
    if (isDigit (_text[_cursor]))
    {
      ++_cursor;
      while (isDigit (_text[_cursor]))
        ++_cursor;

      if (_text[_cursor] == '.')
        ++_cursor;
      else
      {
        _cursor = marker;
        return false;
      }
    }
  }

  if (! isOperator (extractedToken, extractedType) &&
      isIdentifier (extractedToken, extractedType))
  {
    while (1)
    {
      if (_text[_cursor] == '.')
        ++_cursor;
      else
        break;

      if (isOperator (extractedToken, extractedType) ||
          ! isIdentifier (extractedToken, extractedType))
      {
        _cursor = marker;
        return false;
      }
    }

    type = Lexer::Type::dom;
    token = _text.substr (marker, _cursor - marker);
    return true;
  }

  _cursor = marker;
  return false;
}
示例#2
0
static const unsigned char *parseIdentifier (
		const unsigned char *cp, vString *const identifier)
{
	boolean stringLit = FALSE;
	vStringClear (identifier);
	while (*cp != '\0'  &&  (!isIdentifier ((int) *cp) || stringLit))
	{
		int oneback = *cp;
		cp++;
		if (oneback == '('  &&  *cp == '*'  &&  stringLit == FALSE)
		{
			CommentLevel++;
			return ++cp;
		}
		if (*cp == '"' && oneback != '\\')
		{
			stringLit = TRUE;
			continue;
		}
		if (stringLit && *cp == '"' && oneback != '\\')
			stringLit = FALSE;
	}
	if (strcmp ((const char *) cp, "") == 0 || cp == NULL)
		return cp;

	while (isIdentifier ((int) *cp))
	{
		vStringPut (identifier, (int) *cp);
		cp++;
	}
	vStringTerminate (identifier);
	return cp;
}
示例#3
0
const Token* AttributesParser::parseAttributes(const Token* pNext)
{
	pNext = parseAttribute(pNext);
	while (isOperator(pNext, OperatorToken::OP_COMMA) || isIdentifier(pNext))
	{
		if (!isIdentifier(pNext)) pNext = next();
		pNext = parseAttribute(pNext);
	}
	return pNext;
}
示例#4
0
static bool isTableName(const std::string& str) {
	if(isIdentifier(str)){
		return true;
	} else {
		if(str.size() >= 3 && str[0] == '"' && str[str.size()-1]=='"' &&
				isIdentifier(str.substr(1, str.size()-2))){
			return true;
		}
	}
	return false;
}
static std::string parseIdentifier( std::string::const_iterator& si, const std::string::const_iterator& se, const char* idname)
{
	si = skipSpaces( si, se);
	if (!isIdentifier( *si))
	{
		std::string str( si, se);
		throw strus::runtime_error( _TXT( "identifier (%s) expected at '%s'"), idname, str.c_str());
	}
	std::string rt;
	for (; si != se && isIdentifier( *si); ++si)
	{
		rt.push_back( *si);
	}
	return rt;
}
示例#6
0
文件: TlpLexer.cpp 项目: lncosie/ogdf
bool Lexer::tokenizeLine()
{
	while(m_begin != m_end && isspace(*m_begin)) {
		++m_begin;
	}

	// We got an end of a line or a comment.
	if(m_begin == m_end || *m_begin == ';') {
		return true;
	}

	if(*m_begin == '(') {
		m_tokens.push_back(Token(Token::tok_leftParen, line(), column()));
		++m_begin;
		return tokenizeLine();
	}

	if(*m_begin == ')') {
		m_tokens.push_back(Token(Token::tok_rightParen, line(), column()));
		++m_begin;
		return tokenizeLine();
	}

	if(*m_begin == '"') {
		return tokenizeString() && tokenizeLine();
	}

	if(isIdentifier(*m_begin)) {
		return tokenizeIdentifier() && tokenizeLine();
	}

	OGDF_ERROR("Unexpected character \"" << *m_begin << "\" at (" << line() << ", " << column() << ").");
	return false;
}
示例#7
0
////////////////////////////////////////////////////////////////////////////////
// Lexer::Type::pair
//   <identifier> <separator> [ <string> | <word> ]
//   separator '::' | ':=' | ':' | '='
bool Lexer::isPair (std::string& token, Lexer::Type& type)
{
  std::size_t marker = _cursor;

  std::string ignoredToken;
  Lexer::Type ignoredType;
  if (isIdentifier (ignoredToken, ignoredType))
  {
    // Look for a valid separator.
    std::string separator = _text.substr (_cursor, 2);
    if (separator == "::" || separator == ":=")
      _cursor += 2;
    else if (separator[0] == ':' || separator[0] == '=')
      _cursor++;
    else
    {
      _cursor = marker;
      return false;
    }

    // String, word or nothing are all valid.
    if (readWord (_text, "'\"", _cursor, ignoredToken) ||
        readWord (_text,        _cursor, ignoredToken) ||
        isEOS ()                                       ||
        isWhitespace (_text[_cursor]))
    {
      token = _text.substr (marker, _cursor - marker);
      type = Lexer::Type::pair;
      return true;
    }
  }

  _cursor = marker;
  return false;
}
示例#8
0
文件: perl.c 项目: relaxdiego/ctags
/* `end' points to the equal sign.  Parse from right to left to get the
 * identifier.  Assume we're dealing with something of form \s*\w+\s*=>
 */
static void makeTagFromLeftSide (const char *begin, const char *end,
	vString *name, vString *package)
{
	tagEntryInfo entry;
	const char *b, *e;
	if (! PerlKinds[K_CONSTANT].enabled)
		return;
	for (e = end - 1; e > begin && isspace(*e); --e)
		;
	if (e < begin)
		return;
	for (b = e; b >= begin && isIdentifier(*b); --b)
		;
	/* Identifier must be either beginning of line of have some whitespace
	 * on its left:
	 */
	if (b < begin || isspace(*b) || ',' == *b)
		++b;
	else if (b != begin)
		return;
	if (e - b + 1 <= 0)
		return;			/* Left side of => has an invalid identifier. */
	vStringClear(name);
	vStringNCatS(name, b, e - b + 1);
	initTagEntry(&entry, vStringValue(name), &(PerlKinds[K_CONSTANT]));
	makeTagEntry(&entry);
	if (Option.include.qualifiedTags && package && vStringLength(package)) {
		vStringClear(name);
		vStringCopy(name, package);
		vStringNCatS(name, b, e - b + 1);
		initTagEntry(&entry, vStringValue(name), &(PerlKinds[K_CONSTANT]));
		makeTagEntry(&entry);
	}
}
示例#9
0
文件: Fsm.cpp 项目: nikehat/Compiler
void Fsm::identify(Token& t) {
	if (t.type() == "") {
		if (isIdentifier(t.lexeme())) { t.type("identifier"); }
		else if (isRealNumber(t.lexeme())) { t.type("real"); }
		else if (isInteger(t.lexeme())) { t.type("integer"); }
		else { t.type("unknown"); }
	}
}
示例#10
0
文件: perl.c 项目: pjkack/ctags
/*
 * Perl subroutine declaration may look like one of the following:
 *
 *  sub abc;
 *  sub abc :attr;
 *  sub abc (proto);
 *  sub abc (proto) :attr;
 *
 * Note that there may be more than one attribute.  Attributes may
 * have things in parentheses (they look like arguments).  Anything
 * inside of those parentheses goes.  Prototypes may contain semi-colons.
 * The matching end when we encounter (outside of any parentheses) either
 * a semi-colon (that'd be a declaration) or an left curly brace
 * (definition).
 *
 * This is pretty complicated parsing (plus we all know that only perl can
 * parse Perl), so we are only promising best effort here.
 *
 * If we can't determine what this is (due to a file ending, for example),
 * we will return false.
 */
static bool isSubroutineDeclaration (const unsigned char *cp)
{
	bool attr = false;
	int nparens = 0;

	do {
		for ( ; *cp; ++cp) {
SUB_DECL_SWITCH:
			switch (*cp) {
				case ':':
					if (nparens)
						break;
					else if (true == attr)
						return false;    /* Invalid attribute name */
					else
						attr = true;
					break;
				case '(':
					++nparens;
					break;
				case ')':
					--nparens;
					break;
				case ' ':
				case '\t':
					break;
				case ';':
					if (!nparens)
						return true;
				case '{':
					if (!nparens)
						return false;
				default:
					if (attr) {
						if (isIdentifier1(*cp)) {
							cp++;
							while (isIdentifier (*cp))
								cp++;
							attr = false;
							goto SUB_DECL_SWITCH; /* Instead of --cp; */
						} else {
							return false;
						}
					} else if (nparens) {
						break;
					} else {
						return false;
					}
			}
		}
	} while (NULL != (cp = readLineFromInputFile ()));

	return false;
}
示例#11
0
bool SpecificationParser::processClassDefinition(const std::string & s) {
    //class = 6 chars
    std::string clz = s.substr(6);
    boost::trim(clz);
    if(!isIdentifier(clz)) {
        lastError = "Class >>" + clz + "<< doesn't seem to be an identifier.";
        return false;
    }
    spec.getBeans().push_back(BeanClazzSpecification(clz));
    return true;
}
示例#12
0
文件: perl.c 项目: relaxdiego/ctags
/*
 * Perl subroutine declaration may look like one of the following:
 *
 *  sub abc;
 *  sub abc :attr;
 *  sub abc (proto);
 *  sub abc (proto) :attr;
 *
 * Note that there may be more than one attribute.  Attributes may
 * have things in parentheses (they look like arguments).  Anything
 * inside of those parentheses goes.  Prototypes may contain semi-colons.
 * The matching end when we encounter (outside of any parentheses) either
 * a semi-colon (that'd be a declaration) or an left curly brace
 * (definition).
 *
 * This is pretty complicated parsing (plus we all know that only perl can
 * parse Perl), so we are only promising best effort here.
 *
 * If we can't determine what this is (due to a file ending, for example),
 * we will return FALSE.
 */
static boolean isSubroutineDeclaration (const unsigned char *cp)
{
	boolean attr = FALSE;
	int nparens = 0;

	do {
		for ( ; *cp; ++cp) {
SUB_DECL_SWITCH:
			switch (*cp) {
				case ':':
					if (nparens)
						break;
					else if (TRUE == attr)
						return FALSE;    /* Invalid attribute name */
					else
						attr = TRUE;
					break;
				case '(':
					++nparens;
					break;
				case ')':
					--nparens;
					break;
				case ' ':
				case '\t':
					break;
				case ';':
					if (!nparens)
						return TRUE;
				case '{':
					if (!nparens)
						return FALSE;
				default:
					if (attr) {
						if (isIdentifier1(*cp)) {
							cp++;
							while (isIdentifier (*cp))
								cp++;
							attr = FALSE;
							goto SUB_DECL_SWITCH; /* Instead of --cp; */
						} else {
							return FALSE;
						}
					} else if (nparens) {
						break;
					} else {
						return FALSE;
					}
			}
		}
	} while (NULL != (cp = fileReadLine ()));

	return FALSE;
}
示例#13
0
文件: sig.c 项目: aosm/boot
void
parseLine(char *line)
{
    char *paren, *parenEnd;
    char *ident, *identEnd;
    char *arglist, *arglistEnd;
    char *function;
    
    paren = strchr(line, '(');
    if (paren == NULL)
	goto syntax_error;
    for (identEnd = paren - 1; !isIdentifier(*identEnd); identEnd--)
	continue;
    for (ident = identEnd; isIdentifier(*ident); ident--)
	continue;
    ident++;
    *++identEnd = '\0';
    paren++;
    parenEnd = strchr(paren, ')');
    if (parenEnd == NULL)
	goto syntax_error;
    *parenEnd = '\0';

    arglist = parenEnd + 1;
    while (isspace(*arglist))
	arglist++;
    arglistEnd = strchr(arglist, ';');
    if (arglistEnd == NULL)
	goto syntax_error;
    *arglistEnd = '\0';
    
    function = strdup(ident);
    *ident = '\0';
    outputLine(line, function, paren, arglist);
    free(function);
    return;

syntax_error:
    fprintf(stderr, "Syntax error at line %d\n",lineNumber);
    return;
}
示例#14
0
文件: TlpLexer.cpp 项目: lncosie/ogdf
bool Lexer::tokenizeIdentifier()
{
	Token token(Token::tok_identifier, line(), column());

	while(m_begin != m_end && isIdentifier(*m_begin)) {
		*(token.value) += *m_begin;
		++m_begin;
	}

	m_tokens.push_back(token);
	return true;
}
示例#15
0
bool SpecificationParser::processFieldDefinition(const std::string & s) {
    if(spec.getBeans().empty()) {
        lastError = "Field definition without a class specification? Here: >>" + s + "<<";
        return false;
    }
    std::vector<std::string> strs;
    boost::split(strs, s, boost::is_any_of(" \t"));
    if(strs.empty() || strs.size() == 1) {
        lastError = "Invalid field definition >>" + s + "<<";
        return false;
    }
    strs = clearEmptyStrings(strs);
    //first string is the identifier, so:
    std::string ident = strs.at(0);
    if(!isIdentifier(ident)) {
        lastError = "Field >>" + ident + "<< doesn't seem to be an identifier.";
        return false;
    }
    //second: type or flags ?
    unsigned int concatFrom = 1; //no flags
    unsigned int flags = 0;
    std::string mod_or_type = strs.at(1);
    if(mod_or_type.length() > 0 && (mod_or_type.at(0) == '*' || mod_or_type.at(0) == '+')) {
        char * p = const_cast<char *>(mod_or_type.c_str());
        concatFrom = 2;
        while(*p) {
            switch(*p) {
            case '*':
                flags |= F_POINTER;
                break;
            case '+':
                flags |= F_VECTOR;
                break;
            default:
                lastError = "Invalid field modifier >>" + mod_or_type + "<< doesn't seem to be valid (+,*,+*)";
                return false;
            }
            ++p;
        }
    }
    std::string thetype;
    for(unsigned int i = concatFrom; i < strs.size(); ++i) {
        thetype += strs.at(i) + " ";
    }
    boost::trim(thetype);
    if(thetype == "") {
        //no type specified?
        lastError = "No type specified? Invalid field definition >>" + s + "<<";
        return false;
    }
    spec.getLastBean().getFields().push_back(FieldSpecification(ident, thetype, flags));
    return true;
}
示例#16
0
const Token* AttributesParser::parseIdentifier(const Token* pNext, std::string& id)
{
	if (isIdentifier(pNext))
	{
		id = pNext->asString();
		pNext = next();
		while (isOperator(pNext, OperatorToken::OP_PERIOD))
		{
			id.append(".");
			pNext = next();
			if (isIdentifier(pNext))
			{
				id.append(pNext->asString());
				pNext = next();
			}
			else throw SyntaxException("identifier expected");
		}
		return pNext;
	}
	else throw SyntaxException("identifier expected");
}
示例#17
0
文件: make.c 项目: ajitvin/v
static void readIdentifier (const int first, vString *const id)
{
	int c = first;
	vStringClear (id);
	while (isIdentifier (c))
	{
		vStringPut (id, c);
		c = nextChar ();
	}
	fileUngetc (c);
	vStringTerminate (id);
}
示例#18
0
文件: lex.c 项目: umanghome/cd-sem7
int identifyIdentifiers () {
	FILE *fp;

	fp = fopen("input.tmp", "r");
	if (fp == NULL) {
		printf("Could not open input file.\n");
		return 1;
	}

	char str[1000];
	char c;
	int nextIsNew = true;
	while ((c = getc(fp)) != EOF) {
		if (nextIsNew) clearStr(str);
	
		int counter = 0;

		bool withinString = false;

		while (true) {
			char prevChar = '\0';
			if (c == '"') {
				while (true) {
					prevChar = c;
					c = getc(fp);
					if (c == '"' && prevChar != '\\') {
						break;
					} else {
						continue;
					}
				}
				break;
			}

			if (!isDigit(c) && !isAlpha(c) && c != '_') {
				break;
			} else {
				str[counter++] = c;
				c = getc(fp);
			}
		}

		if (isIdentifier(str)) {
			printf("%s\n", str);
		}
	}

	fclose(fp);

	return 0;

}
示例#19
0
bool SpecificationParser::processNamespace(const std::string & s) {
    if(spec.getBeans().size() > 0) {
        lastError = "Namespace should be first, don't mix them";
        return false;
    }
    //namespace = 10 chars
    std::string nms = s.substr(10);
    boost::trim(nms);
    if(!isIdentifier(nms)) {
        lastError = "Namespace >>" + nms + "<< doesn't seem to be an identifier.";
        return false;
    }
    spec.getNamespaces().push_back(nms);
    return true;
}
示例#20
0
////////////////////////////////////////////////////////////////////////////////
// Lexer::Type::pair
//   <identifier> :|= [ <string> | <word> ]
bool Lexer::isPair (std::string& token, Lexer::Type& type)
{
  std::size_t marker = _cursor;

  std::string ignoredToken;
  Lexer::Type ignoredType;
  if (isIdentifier (ignoredToken, ignoredType))
  {
    // Look for rc.name[:=]value first, because '=' is allowed.
    if (ignoredToken == "rc" ||
        ignoredToken.substr (0, 3) == "rc.")
    {
      if (_eos - _cursor > 1 &&
          (_text[_cursor] == ':' || _text[_cursor] == '='))
      {
        _cursor++;

        if (isString (ignoredToken, ignoredType, '\'') ||
            isString (ignoredToken, ignoredType, '"')  ||
            isWord   (ignoredToken, ignoredType))
        {
          token = _text.substr (marker, _cursor - marker);
          type = Lexer::Type::pair;
          return true;
        }
      }
    }

    if (_eos - _cursor > 1 &&
        _text[_cursor] == ':')
    {
      _cursor++;

      if (isString (ignoredToken, ignoredType, '\'') ||
          isString (ignoredToken, ignoredType, '"')  ||
          isWord   (ignoredToken, ignoredType))
      {
        token = _text.substr (marker, _cursor - marker);
        type = Lexer::Type::pair;
        return true;
      }
    }
  }

  _cursor = marker;
  return false;
}
示例#21
0
文件: Errors.cpp 项目: Saisi/swift
static bool
demangledLinePrefix(std::string line, std::string prefix,
                    std::string &out,
                    bool (*demangle)(std::string, std::string &))
{
  int symbolStart = -1;
  int symbolEnd = -1;
  
  for (size_t i = 0; i < line.size(); i++) {
    char c = line[i];
    
    bool hasBegun = symbolStart != -1;
    bool isIdentifierChar = isIdentifier(c);
    bool isEndOfSymbol = hasBegun && !isIdentifierChar;
    bool isEndOfLine = i == line.size() - 1;
    
    if (isEndOfLine || isEndOfSymbol) {
      symbolEnd = i;
      break;
    }

    bool canFindPrefix = (line.size() - 2) - i > prefix.size();
    if (!hasBegun && canFindPrefix && !isIdentifierChar &&
        line.substr(i + 1, prefix.size()) == prefix) {
      symbolStart = i + 1;
      continue;
    }
  }
  
  if (symbolStart == -1 || symbolEnd == -1) {
    out = line;
    return false;
  } else {
    auto symbol = line.substr(symbolStart, symbolEnd - symbolStart);
    
    std::string demangled;
    bool success = demangle(symbol, demangled);
    
    if (success) {
      line.replace(symbolStart, symbolEnd - symbolStart, demangled);
    }
    
    out = line;
    
    return success;
  }
}
示例#22
0
文件: make.c 项目: jonthn/ctags
static void readIdentifier (const int first, vString *const id)
{
    int depth = 0;
    int c = first;
    vStringClear (id);
    while (isIdentifier (c) || (depth > 0 && c != EOF && c != '\n'))
    {
        if (c == '(' || c == '}')
            depth++;
        else if (depth > 0 && (c == ')' || c == '}'))
            depth--;
        vStringPut (id, c);
        c = nextChar ();
    }
    ungetcToInputFile (c);
    vStringTerminate (id);
}
示例#23
0
文件: bbp.c 项目: sekcheong/monetdb
str
CMDbbpbind(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
	str name;
	ValPtr lhs;
	bat i;
	int ht,tt;
	BAT *b;

	(void) cntxt;
	(void) mb;		/* fool compiler */
	lhs = &stk->stk[pci->argv[0]];
	name = *getArgReference_str(stk, pci, 1);
	if (isIdentifier(name) < 0)
		throw(MAL, "bbp.bind", IDENTIFIER_EXPECTED);
	i = BBPindex(name);
	if (i == 0)
		throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING);
	/* make sure you load the descriptors and heaps */
	b = (BAT *) BATdescriptor(i);
	if (b == 0)
		/* Simple ignore the binding if you can't find the bat */
		throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING);

	/* check conformity of the actual type and the one requested */
	ht= getHeadType(getArgType(mb,pci,0));
	tt= getColumnType(getArgType(mb,pci,0));
	if( b->htype == TYPE_void && ht== TYPE_oid) ht= TYPE_void;
	if( b->ttype == TYPE_void && tt== TYPE_oid) tt= TYPE_void;

	if( ht != b->htype || tt != b->ttype){
		BBPunfix(i);
		throw(MAL, "bbp.bind", SEMANTIC_TYPE_MISMATCH );
	}
	/* make sure we are not dealing with an about to be deleted bat */
	if( BBP_refs(b->batCacheid) == 1 &&
		BBP_lrefs(b->batCacheid) == 0){
		BBPunfix(i);
		throw(MAL, "bbp.bind", RUNTIME_OBJECT_MISSING);
	}

	BBPkeepref(b->batCacheid);
	lhs->vtype = TYPE_bat;
	lhs->val.bval = i;
	return MAL_SUCCEED;
}
示例#24
0
////////////////////////////////////////////////////////////////////////////////
// When a Lexer object is constructed with a string, this method walks through
// the stream of low-level tokens.
bool Lexer::token (std::string& token, Lexer::Type& type)
{
  // Eat white space.
  while (isWhitespace (_text[_cursor]))
    utf8_next_char (_text, _cursor);

  // Terminate at EOS.
  if (isEOS ())
    return false;

  // The sequence is specific, and must follow these rules:
  //   - date < duration < uuid < identifier
  //   - dom < uuid
  //   - uuid < hex < number
  //   - url < pair < identifier
  //   - hex < number
  //   - separator < tag < operator
  //   - path < substitution < pattern
  //   - set < number
  //   - word last
  if (isString       (token, type, "'\"") ||
      isDate         (token, type)        ||
      isDuration     (token, type)        ||
      isURL          (token, type)        ||
      isPair         (token, type)        ||
      isUUID         (token, type, true)  ||
      isSet          (token, type)        ||
      isDOM          (token, type)        ||
      isHexNumber    (token, type)        ||
      isNumber       (token, type)        ||
      isSeparator    (token, type)        ||
      isTag          (token, type)        ||
      isPath         (token, type)        ||
      isSubstitution (token, type)        ||
      isPattern      (token, type)        ||
      isOperator     (token, type)        ||
      isIdentifier   (token, type)        ||
      isWord         (token, type))
    return true;

  return false;
}
示例#25
0
const Token* AttributesParser::parseAttribute(const Token* pNext)
{
	std::string id;
	std::string value;
	pNext = parseIdentifier(pNext, id);
	if (isOperator(pNext, OperatorToken::OP_ASSIGN))
	{
		pNext = next();
		if (isOperator(pNext, OperatorToken::OP_OPENBRACE))
		{
			pNext = parseComplexAttribute(pNext, id);
		}
		else if (isIdentifier(pNext) || isLiteral(pNext))
		{
			value = pNext->asString();
			pNext = next();
		}
		else throw SyntaxException("bad attribute declaration");
	}
	setAttribute(id, value);
	return pNext;
}
UStringImpl::~UStringImpl()
{
    ASSERT(!isStatic());

    if (isIdentifier())
        Identifier::remove(this);

    BufferOwnership ownership = bufferOwnership();
    if (ownership != BufferInternal) {
        if (ownership == BufferOwned) {
            ASSERT(!m_sharedBuffer);
            ASSERT(m_data);
            fastFree(const_cast<UChar*>(m_data));
        } else if (ownership == BufferSubstring) {
            ASSERT(m_substringBuffer);
            m_substringBuffer->deref();
        } else {
            ASSERT(ownership == BufferShared);
            ASSERT(m_sharedBuffer);
            m_sharedBuffer->deref();
        }
    }
}
示例#27
0
文件: make.c 项目: jonthn/ctags
static void findMakeTags (void)
{
    stringList *identifiers = stringListNew ();
    boolean newline = TRUE;
    boolean in_define = FALSE;
    boolean in_rule = FALSE;
    boolean variable_possible = TRUE;
    int c;

    while ((c = nextChar ()) != EOF)
    {
        if (newline)
        {
            if (in_rule)
            {
                if (c == '\t' || (c = skipToNonWhite (c)) == '#')
                {
                    skipLine ();  /* skip rule or comment */
                    c = nextChar ();
                }
                else if (c != '\n')
                    in_rule = FALSE;
            }
            stringListClear (identifiers);
            variable_possible = (boolean)(!in_rule);
            newline = FALSE;
        }
        if (c == '\n')
            newline = TRUE;
        else if (isspace (c))
            continue;
        else if (c == '#')
            skipLine ();
        else if (variable_possible && c == '?')
        {
            c = nextChar ();
            ungetcToInputFile (c);
            variable_possible = (c == '=');
        }
        else if (variable_possible && c == ':' &&
                 stringListCount (identifiers) > 0)
        {
            c = nextChar ();
            ungetcToInputFile (c);
            if (c != '=')
            {
                unsigned int i;
                for (i = 0; i < stringListCount (identifiers); i++)
                    newTarget (stringListItem (identifiers, i));
                stringListClear (identifiers);
                in_rule = TRUE;
            }
        }
        else if (variable_possible && c == '=' &&
                 stringListCount (identifiers) == 1)
        {
            newMacro (stringListItem (identifiers, 0));
            skipLine ();
            in_rule = FALSE;
        }
        else if (variable_possible && isIdentifier (c))
        {
            vString *name = vStringNew ();
            readIdentifier (c, name);
            stringListAdd (identifiers, name);

            if (stringListCount (identifiers) == 1)
            {
                if (in_define && ! strcmp (vStringValue (name), "endef"))
                    in_define = FALSE;
                else if (in_define)
                    skipLine ();
                else if (! strcmp (vStringValue (name), "define"))
                {
                    in_define = TRUE;
                    c = skipToNonWhite (nextChar ());
                    vStringClear (name);
                    /* all remaining characters on the line are the name -- even spaces */
                    while (c != EOF && c != '\n')
                    {
                        vStringPut (name, c);
                        c = nextChar ();
                    }
                    if (c == '\n')
                        ungetcToInputFile (c);
                    vStringTerminate (name);
                    vStringStripTrailing (name);
                    newMacro (name);
                }
                else if (! strcmp (vStringValue (name), "export"))
                    stringListClear (identifiers);
                else if (! strcmp (vStringValue (name), "include")
                         || ! strcmp (vStringValue (name), "sinclude")
                         || ! strcmp (vStringValue (name), "-include"))
                {
                    boolean optional = (vStringValue (name)[0] == 'i')? FALSE: TRUE;
                    while (1)
                    {
                        c = skipToNonWhite (nextChar ());
                        readIdentifier (c, name);
                        vStringStripTrailing (name);
                        if (isAcceptableAsInclude(name))
                            newInclude (name, optional);

                        /* non-space characters after readIdentifier() may
                         * be rejected by the function:
                         * e.g.
                         * include $*
                         *
                         * Here, remove such characters from input stream.
                         */
                        do
                            c = nextChar ();
                        while (c != EOF && c != '\n' && (!isspace (c)));
                        if (c == '\n')
                            ungetcToInputFile (c);

                        if (c == EOF || c == '\n')
                            break;
                    }
                }
            }
        }
        else
            variable_possible = FALSE;
    }
    stringListDelete (identifiers);
}
示例#28
0
文件: perl.c 项目: relaxdiego/ctags
/* Algorithm adapted from from GNU etags.
 * Perl support by Bart Robinson <*****@*****.**>
 * Perl sub names: look for /^ [ \t\n]sub [ \t\n]+ [^ \t\n{ (]+/
 */
static void findPerlTags (void)
{
	vString *name = vStringNew ();
	vString *package = NULL;
	boolean skipPodDoc = FALSE;
	const unsigned char *line;

	/* Core modules AutoLoader and SelfLoader support delayed compilation
	 * by allowing Perl code that follows __END__ and __DATA__ tokens,
	 * respectively.  When we detect that one of these modules is used
	 * in the file, we continue processing even after we see the
	 * corresponding token that would usually terminate parsing of the
	 * file.
	 */
	enum {
		RESPECT_END		= (1 << 0),
		RESPECT_DATA	= (1 << 1),
	} respect_token = RESPECT_END | RESPECT_DATA;

	while ((line = fileReadLine ()) != NULL)
	{
		boolean spaceRequired = FALSE;
		boolean qualified = FALSE;
		const unsigned char *cp = line;
		perlKind kind = K_NONE;
		tagEntryInfo e;

		if (skipPodDoc)
		{
			if (strncmp ((const char*) line, "=cut", (size_t) 4) == 0)
				skipPodDoc = FALSE;
			continue;
		}
		else if (line [0] == '=')
		{
			skipPodDoc = isPodWord ((const char*)line + 1);
			continue;
		}
		else if (strcmp ((const char*) line, "__DATA__") == 0)
		{
			if (respect_token & RESPECT_DATA)
				break;
			else
				continue;
		}
		else if (strcmp ((const char*) line, "__END__") == 0)
		{
			if (respect_token & RESPECT_END)
				break;
			else
				continue;
		}
		else if (line [0] == '#')
			continue;

		while (isspace (*cp))
			cp++;

		if (strncmp((const char*) cp, "sub", (size_t) 3) == 0)
		{
			TRACE("this looks like a sub\n");
			cp += 3;
			kind = K_SUBROUTINE;
			spaceRequired = TRUE;
			qualified = TRUE;
		}
		else if (strncmp((const char*) cp, "use", (size_t) 3) == 0)
		{
			cp += 3;
			if (!isspace(*cp))
				continue;
			while (*cp && isspace (*cp))
				++cp;
			if (strncmp((const char*) cp, "AutoLoader", (size_t) 10) == 0) {
				respect_token &= ~RESPECT_END;
				continue;
			}
			if (strncmp((const char*) cp, "SelfLoader", (size_t) 10) == 0) {
				respect_token &= ~RESPECT_DATA;
				continue;
			}
			if (strncmp((const char*) cp, "constant", (size_t) 8) != 0)
				continue;
			cp += 8;
			/* Skip up to the first non-space character, skipping empty
			 * and comment lines.
			 */
			while (isspace(*cp))
				cp++;
			while (!*cp || '#' == *cp) {
				cp = fileReadLine ();
				if (!cp)
					goto END_MAIN_WHILE;
				while (isspace (*cp))
					cp++;
			}
			if ('{' == *cp) {
				++cp;
				if (0 == parseConstantsFromHashRef(cp, name, package)) {
					vStringClear(name);
					continue;
				} else
					goto END_MAIN_WHILE;
			}
			kind = K_CONSTANT;
			spaceRequired = FALSE;
			qualified = TRUE;
		}
		else if (strncmp((const char*) cp, "package", (size_t) 7) == 0 &&
				 ('\0' == cp[7] || isspace(cp[7])))
		{
			cp += 7;
			while (isspace (*cp))
				cp++;
			while (!*cp || '#' == *cp) {
				cp = fileReadLine ();
				if (!cp)
					goto END_MAIN_WHILE;
				while (isspace (*cp))
					cp++;
			}
			if (package == NULL)
				package = vStringNew ();
			else
				vStringClear (package);
			const unsigned char *const first = cp;
			while (*cp && (int) *cp != ';'  &&  !isspace ((int) *cp))
			{
				vStringPut (package, (int) *cp);
				cp++;
			}
			vStringCatS (package, "::");

			cp = first;	 /* Rewind */
			kind = K_PACKAGE;
			spaceRequired = FALSE;
			qualified = TRUE;
		}
		else if (strncmp((const char*) cp, "format", (size_t) 6) == 0)
		{
			cp += 6;
			kind = K_FORMAT;
			spaceRequired = TRUE;
			qualified = TRUE;
		}
		else
		{
			if (isIdentifier1 (*cp))
			{
				const unsigned char *p = cp;
				while (isIdentifier (*p))
					++p;
				while (isspace (*p))
					++p;
				if ((int) *p == ':' && (int) *(p + 1) != ':')
					kind = K_LABEL;
			}
		}
		if (kind != K_NONE)
		{
			TRACE("cp0: %s\n", (const char *) cp);
			if (spaceRequired && *cp && !isspace (*cp))
				continue;

			TRACE("cp1: %s\n", (const char *) cp);
			while (isspace (*cp))
				cp++;

			while (!*cp || '#' == *cp) { /* Gobble up empty lines
				                            and comments */
				cp = fileReadLine ();
				if (!cp)
					goto END_MAIN_WHILE;
				while (isspace (*cp))
					cp++;
			}

			while (isIdentifier (*cp) || (K_PACKAGE == kind && ':' == *cp))
			{
				vStringPut (name, (int) *cp);
				cp++;
			}

			if (K_FORMAT == kind &&
				vStringLength (name) == 0 && /* cp did not advance */
				'=' == *cp)
			{
				/* format's name is optional.  If it's omitted, 'STDOUT'
				   is assumed. */
				vStringCatS (name, "STDOUT");
			}

			vStringTerminate (name);
			TRACE("name: %s\n", name->buffer);

			if (0 == vStringLength(name)) {
				vStringClear(name);
				continue;
			}

			if (K_SUBROUTINE == kind)
			{
				/*
				 * isSubroutineDeclaration() may consume several lines.  So
				 * we record line positions.
				 */
				initTagEntry(&e, vStringValue(name), NULL);

				if (TRUE == isSubroutineDeclaration(cp)) {
					if (TRUE == PerlKinds[K_SUBROUTINE_DECLARATION].enabled) {
						kind = K_SUBROUTINE_DECLARATION;
					} else {
						vStringClear (name);
						continue;
					}
				} else if (! PerlKinds[kind].enabled) {
					continue;
				}

				e.kind     = &(PerlKinds[kind]);

				makeTagEntry(&e);

				if (Option.include.qualifiedTags && qualified &&
					package != NULL  && vStringLength (package) > 0)
				{
					vString *const qualifiedName = vStringNew ();
					vStringCopy (qualifiedName, package);
					vStringCat (qualifiedName, name);
					e.name = vStringValue(qualifiedName);
					makeTagEntry(&e);
					vStringDelete (qualifiedName);
				}
			} else if (vStringLength (name) > 0)
			{
				makeSimpleTag (name, PerlKinds, kind);
				if (Option.include.qualifiedTags && qualified &&
					K_PACKAGE != kind &&
					package != NULL  && vStringLength (package) > 0)
				{
					vString *const qualifiedName = vStringNew ();
					vStringCopy (qualifiedName, package);
					vStringCat (qualifiedName, name);
					makeSimpleTag (qualifiedName, PerlKinds, kind);
					vStringDelete (qualifiedName);
				}
			}
			vStringClear (name);
		}
	}

END_MAIN_WHILE:
	vStringDelete (name);
	if (package != NULL)
		vStringDelete (package);
}
示例#29
0
文件: make.c 项目: Monits/ctags
static void findMakeTags (void)
{
	stringList *identifiers = stringListNew ();
	boolean newline = TRUE;
	boolean in_define = FALSE;
	boolean in_rule = FALSE;
	boolean variable_possible = TRUE;
	int c;

	while ((c = nextChar ()) != EOF)
	{
		if (newline)
		{
			if (in_rule)
			{
				if (c == '\t' || (c = skipToNonWhite (c)) == '#')
				{
					skipLine ();  /* skip rule or comment */
					c = nextChar ();
				}
				else if (c != '\n')
					in_rule = FALSE;
			}
			stringListClear (identifiers);
			variable_possible = (boolean)(!in_rule);
			newline = FALSE;
		}
		if (c == '\n')
			newline = TRUE;
		else if (isspace (c))
			continue;
		else if (c == '#')
			skipLine ();
		else if (variable_possible && c == '?')
		{
			c = nextChar ();
			fileUngetc (c);
			variable_possible = (c == '=');
		}
		else if (variable_possible && c == ':' &&
				 stringListCount (identifiers) > 0)
		{
			c = nextChar ();
			fileUngetc (c);
			if (c != '=')
			{
				unsigned int i;
				for (i = 0; i < stringListCount (identifiers); i++)
					newTarget (stringListItem (identifiers, i));
				stringListClear (identifiers);
				in_rule = TRUE;
			}
		}
		else if (variable_possible && c == '=' &&
				 stringListCount (identifiers) == 1)
		{
			newMacro (stringListItem (identifiers, 0));
			skipLine ();
			in_rule = FALSE;
		}
		else if (variable_possible && isIdentifier (c))
		{
			vString *name = vStringNew ();
			readIdentifier (c, name);
			stringListAdd (identifiers, name);

			if (stringListCount (identifiers) == 1)
			{
				if (in_define && ! strcmp (vStringValue (name), "endef"))
					in_define = FALSE;
				else if (in_define)
					skipLine ();
				else if (! strcmp (vStringValue (name), "define"))
				{
					in_define = TRUE;
					c = skipToNonWhite (nextChar ());
					vStringClear (name);
					/* all remaining characters on the line are the name -- even spaces */
					while (c != EOF && c != '\n')
					{
						vStringPut (name, c);
						c = nextChar ();
					}
					if (c == '\n')
						fileUngetc (c);
					vStringTerminate (name);
					vStringStripTrailing (name);
					newMacro (name);
				}
				else if (! strcmp (vStringValue (name), "export"))
					stringListClear (identifiers);
			}
		}
		else
			variable_possible = FALSE;
	}
	stringListDelete (identifiers);
}
示例#30
0
TokenList KoEnhancedPathFormula::scan( const QString &formula ) const
{
  // parsing state
  enum { Start, Finish, Bad, InNumber, InDecimal, InExpIndicator, InExponent,
    InString, InIdentifier } state;

    TokenList tokens;

    int i = 0;
    state = Start;
    int tokenStart = 0;
    QString tokenText;
    QString expr = formula + QChar();

    // main loop
    while( (state != Bad) && (state != Finish) && (i < expr.length() ) )
    {
        QChar ch = expr[i];

        switch( state )
        {
            case Start:
                tokenStart = i;

                // skip any whitespaces
                if( ch.isSpace() )
                {
                    i++;
                }
                // check for number
                else if( ch.isDigit() )
                {
                    state = InNumber;
                }
                // beginning with alphanumeric ?
                // could be identifier, function, function reference, modifier reference
                else if( isIdentifier( ch ) )
                {
                    state = InIdentifier;
                }
                // decimal dot ?
                else if ( ch == '.' )
                {
                    tokenText.append( expr[i++] );
                    state = InDecimal;
                }
                // terminator character
                else if ( ch == QChar::Null )
                {
                    state = Finish;
                }
                // look for operator match
                else
                {
                    QString opString( ch );
                    int op = matchOperator( opString );

                    // any matched operator ?
                    if( op != FormulaToken::OperatorInvalid )
                    {
                        i++;
                        tokens.append( FormulaToken( FormulaToken::TypeOperator, opString, tokenStart ) );
                    }
                    else state = Bad;
                }
            break;
            case InIdentifier:
                // consume as long as alpha, dollar sign, question mark, or digit
                if( isIdentifier( ch ) || ch.isDigit() )
                {
                    tokenText.append( expr[i++] );
                }
                // a '(' ? then this must be a function identifier
                else if( ch == '(' )
                {
                    tokens.append( FormulaToken( FormulaToken::TypeFunction, tokenText, tokenStart) );
                    tokenStart = i;
                    tokenText = "";
                    state = Start;
                }
                // we're done with identifier
                else
                {
                    tokens.append( FormulaToken( FormulaToken::TypeReference, tokenText, tokenStart) );
                    tokenStart = i;
                    tokenText = "";
                    state = Start;
                }
            break;
            case InNumber:
                // consume as long as it's digit
                if( ch.isDigit() )
                {
                    tokenText.append( expr[i++] );
                }
                // decimal dot ?
                else if( ch == '.' )
                {
                    tokenText.append( '.' );
                    i++;
                    state = InDecimal;
                }
                // exponent ?
                else if( ch.toUpper() == 'E' )
                {
                    tokenText.append( 'E' );
                    i++;
                    state = InExpIndicator;
                }
                // we're done with integer number
                else
                {
                    tokens.append( FormulaToken( FormulaToken::TypeNumber, tokenText, tokenStart ) );
                    tokenText = "";
                    state = Start;
                };
            break;
            case InDecimal:
                // consume as long as it's digit
                if( ch.isDigit() )
                {
                    tokenText.append( expr[i++] );
                }
                // exponent ?
                else if( ch.toUpper() == 'E' )
                {
                    tokenText.append( 'E' );
                    i++;
                    state = InExpIndicator;
                }
                // we're done with floating-point number
                else
                {
                    tokens.append( FormulaToken( FormulaToken::TypeNumber, tokenText, tokenStart ) );
                    tokenText = "";
                    state = Start;
                };
            break;
            case InExpIndicator:
                // possible + or - right after E, e.g 1.23E+12 or 4.67E-8
                if( ( ch == '+' ) || ( ch == '-' ) )
                {
                    tokenText.append( expr[i++] );
                }
                // consume as long as it's digit
                else if( ch.isDigit() )
                {
                    state = InExponent;
                }
                // invalid thing here
                else state = Bad;
            break;
            case InExponent:
                // consume as long as it's digit
                if( ch.isDigit() )
                {
                    tokenText.append( expr[i++] );
                }
                // we're done with floating-point number
                else
                {
                    tokens.append( FormulaToken( FormulaToken::TypeNumber, tokenText, tokenStart ) );
                    tokenText = "";
                    state = Start;
                };
                break;
            case Bad:
            default:
            break;
        }
    }
     return tokens;
}