Example #1
0
int
pdf_lex(fz_stream *f, pdf_lexbuf *buf)
{
	while (1)
	{
		int c = fz_read_byte(f);
		switch (c)
		{
		case EOF:
			return PDF_TOK_EOF;
		case IS_WHITE:
			lex_white(f);
			break;
		case '%':
			lex_comment(f);
			break;
		case '/':
			lex_name(f, buf);
			return PDF_TOK_NAME;
		case '(':
			return lex_string(f, buf);
		case ')':
			fz_warn(f->ctx, "lexical error (unexpected ')')");
			continue;
		case '<':
			c = fz_read_byte(f);
			if (c == '<')
			{
				return PDF_TOK_OPEN_DICT;
			}
			else
			{
				fz_unread_byte(f);
				return lex_hex_string(f, buf);
			}
		case '>':
			c = fz_read_byte(f);
			if (c == '>')
			{
				return PDF_TOK_CLOSE_DICT;
			}
			fz_warn(f->ctx, "lexical error (unexpected '>')");
			continue;
		case '[':
			return PDF_TOK_OPEN_ARRAY;
		case ']':
			return PDF_TOK_CLOSE_ARRAY;
		case '{':
			return PDF_TOK_OPEN_BRACE;
		case '}':
			return PDF_TOK_CLOSE_BRACE;
		case IS_NUMBER:
			return lex_number(f, buf, c);
		default: /* isregular: !isdelim && !iswhite && c != EOF */
			fz_unread_byte(f);
			lex_name(f, buf);
			return pdf_token_from_keyword(buf->scratch);
		}
	}
}
Example #2
0
Token
Lexer::lex()
{
    assert(!cs.eof());

    while (!cs.eof()) {
        switch (cs.peek()) {
        case '\n':
            std::cout << "Error: unexpected newline \n";
            cs.get();
            return Token(-1, error_tok, "error");
        // if its a space, consume and continue
        case ' ':
            cs.get();
            break;
        // if its a single character token
        case '+':
            return lex_char(cs.location(), plus_tok, cs.get(), 1);
        case '-':
            return lex_char(cs.location(), minus_tok, cs.get(), 1);
        case '*':
            return lex_char(cs.location(), star_tok, cs.get(), 1);
        case '/':
            return lex_char(cs.location(), fslash_tok, cs.get(), 1);
        case '%':
            return lex_char(cs.location(), mod_tok, cs.get(), 1);
        case '(':
            return lex_char(cs.location(), lparen_tok, cs.get(), 1);
        case ')':
            return lex_char(cs.location(), rparen_tok, cs.get(), 1);
        default:
            // if its none of those then it has to be a number or identifier
            if (is_digit(cs.peek()))
                return lex_number(cs.location(), cs);
            else {
                std::cout << "ERROR: unrecognized token: " << cs.get() << '\n';
                return Token(-1, error_tok, "error");
            }
        }
    }

    // handles trailing spaces which don't end in anything
    return Token(-2, error_tok, "eof");
}
Example #3
0
Token extract_next_token(std::istream& stream) {
    char c;
    while (stream.get(c) && std::isspace(c));

    if (!stream)
        return {end_of_file_token, ""};

    if (c == '(')
        return {open_paren_token, "("};
    if (c == ')')
        return {close_paren_token, ")"};

    stream.unget();

    if (std::isalpha(c))
        return lex_name(stream);
    if (std::isdigit(c))
        return lex_number(stream);
    if (isoperator(c))
        return lex_operator(stream);

    throw std::runtime_error{"unrecognised character"};
}
Example #4
0
std::list<Token> jsonnet_lex(const std::string &filename, const char *input)
{
    unsigned long line_number = 1;
    const char *line_start = input;

    std::list<Token> r;

    const char *c = input;

    for ( ; *c!='\0' ; ++c) {
        Location begin(line_number, c - line_start + 1);
        Token::Kind kind;
        std::string data;

        switch (*c) {

            // Skip non-\n whitespace
            case ' ': case '\t': case '\r':
            continue;

            // Skip \n and maintain line numbers
            case '\n':
            line_number++;
            line_start = c+1;
            continue;

            case '{':
            kind = Token::BRACE_L;
            break;

            case '}':
            kind = Token::BRACE_R;
            break;

            case '[':
            kind = Token::BRACKET_L;
            break;

            case ']':
            kind = Token::BRACKET_R;
            break;

            case ':':
            kind = Token::COLON;
            break;

            case ',':
            kind = Token::COMMA;
            break;

            case '$':
            kind = Token::DOLLAR;
            break;

            case '.':
            kind = Token::DOT;
            break;

            case '(':
            kind = Token::PAREN_L;
            break;

            case ')':
            kind = Token::PAREN_R;
            break;

            case ';':
            kind = Token::SEMICOLON;
            break;

            // Special cases for unary operators.
            case '!':
            kind = Token::OPERATOR;
            if (*(c+1) == '=') {
                c++;
                data = "!=";
            } else {
                data = "!";
            }
            break;

            case '~':
            kind = Token::OPERATOR;
            data = "~";
            break;

            case '+':
            kind = Token::OPERATOR;
            data = "+";

            break;
            case '-':
            kind = Token::OPERATOR;
            data = "-";
            break;

            // Numeric literals.
            case '0': case '1': case '2': case '3': case '4':
            case '5': case '6': case '7': case '8': case '9':
            kind = Token::NUMBER;
            data = lex_number(c, filename, begin);
            break;

            // String literals.
            case '"': {
                c++;
                for (; ; ++c) {
                    if (*c == '\0') {
                        throw StaticError(filename, begin, "Unterminated string");
                    }
                    if (*c == '"') {
                        break;
                    }
                    switch (*c) {
                        case '\\':
                        switch (*(++c)) {
                            case '"':
                            data += *c;
                            break;

                            case '\\':
                            data += *c;
                            break;

                            case '/':
                            data += *c;
                            break;

                            case 'b':
                            data += '\b';
                            break;

                            case 'f':
                            data += '\f';
                            break;

                            case 'n':
                            data += '\n';
                            break;

                            case 'r':
                            data += '\r';
                            break;

                            case 't':
                            data += '\t';
                            break;

                            case 'u': {
                                ++c;  // Consume the 'u'.
                                unsigned long codepoint = 0;
                                // Expect 4 hex digits.
                                for (unsigned i=0 ; i<4 ; ++i) {
                                    auto x = (unsigned char)(c[i]);
                                    unsigned digit;
                                    if (x == '\0') {
                                        auto msg = "Unterminated string";
                                        throw StaticError(filename, begin, msg);
                                    } else if (x == '"') {
                                        auto msg = "Truncated unicode escape sequence in "
                                                   "string literal.";
                                        throw StaticError(filename, begin, msg);
                                    } else if (x >= '0' && x <= '9') {
                                        digit = x - '0';
                                    } else if (x >= 'a' && x <= 'f') {
                                        digit = x - 'a' + 10;
                                    } else if (x >= 'A' && x <= 'F') {
                                        digit = x - 'A' + 10;
                                    } else {
                                        std::stringstream ss;
                                        ss << "Malformed unicode escape character, "
                                           << "should be hex: '" << x << "'";
                                        throw StaticError(filename, begin, ss.str());
                                    }
                                    codepoint *= 16;
                                    codepoint += digit;
                                }

                                encode_utf8(codepoint, data);

                                // Leave us on the last char, ready for the ++c at
                                // the outer for loop.
                                c += 3;
                            }
                            break;

                            case '\0': {
                                auto msg = "Truncated escape sequence in string literal.";
                                throw StaticError(filename, begin, msg);
                            }

                            default: {
                                std::stringstream ss;
                                ss << "Unknown escape sequence in string literal: '" << *c << "'";
                                throw StaticError(filename, begin, ss.str());
                            }
                        }
                        break;

                        // Treat as a regular letter, but maintain line/column counters.
                        case '\n':
                        line_number++;
                        line_start = c+1;
                        data += *c;
                        break;

                        default:
                        // Just a regular letter.
                        data += *c;
                    }
                }
                kind = Token::STRING;
            }
            break;

            // Keywords
            default:
            if (is_identifier_first(*c)) {
                std::string id;
                for (; *c != '\0' ; ++c) {
                    if (!is_identifier(*c)) {
                        break;
                    }
                    id += *c;
                }
                --c;
                if (id == "assert") {
                    kind = Token::ASSERT;
                } else if (id == "else") {
                    kind = Token::ELSE;
                } else if (id == "error") {
                    kind = Token::ERROR;
                } else if (id == "false") {
                    kind = Token::FALSE;
                } else if (id == "for") {
                    kind = Token::FOR;
                } else if (id == "function") {
                    kind = Token::FUNCTION;
                } else if (id == "if") {
                    kind = Token::IF;
                } else if (id == "import") {
                    kind = Token::IMPORT;
                } else if (id == "importstr") {
                    kind = Token::IMPORTSTR;
                } else if (id == "in") {
                    kind = Token::IN;
                } else if (id == "local") {
                    kind = Token::LOCAL;
                } else if (id == "null") {
                    kind = Token::NULL_LIT;
                } else if (id == "self") {
                    kind = Token::SELF;
                } else if (id == "super") {
                    kind = Token::SUPER;
                } else if (id == "tailstrict") {
                    kind = Token::TAILSTRICT;
                } else if (id == "then") {
                    kind = Token::THEN;
                } else if (id == "true") {
                    kind = Token::TRUE;
                } else {
                    // Not a keyword, must be an identifier.
                    kind = Token::IDENTIFIER;
                    data = id;
                }
            } else if (is_symbol(*c)) {

                // Single line C++ style comment
                if (*c == '/' && *(c+1) == '/') {
                    while (*c != '\0' && *c != '\n') {
                        ++c;
                    }
                    // Leaving it on the \n allows processing of \n on next iteration,
                    // i.e. managing of the line & column counter.
                    c--;
                    continue;
                }

                // Single line # comment
                if (*c == '#') {
                    while (*c != '\0' && *c != '\n') {
                        ++c;
                    }
                    // Leaving it on the \n allows processing of \n on next iteration,
                    // i.e. managing of the line & column counter.
                    c--;
                    continue;
                }

                // Multi-line comment.
                if (*c == '/' && *(c+1) == '*') {
                    c += 2;  // Avoid matching /*/: skip the /* before starting the search for */.
                    while (*c != '\0' && !(*c == '*' && *(c+1) == '/')) {
                        if (*c == '\n') {
                            // Just keep track of the line / column counters.
                            line_number++;
                            line_start = c+1;
                        }
                        ++c;
                    }
                    if (*c == '\0') {
                        auto msg = "Multi-line comment has no terminating */.";
                        throw StaticError(filename, begin, msg);
                    }
                    // Leave the counter on the closing /.
                    c++;
                    continue;
                }
                // Text block
                if (*c == '|' && *(c+1) == '|' && *(c+2) == '|' && *(c+3) == '\n') {
                    std::stringstream block;
                    c += 4; // Skip the "|||\n"
                    line_number++;
                    line_start = c;
                    const char *first_line = c;
                    int ws_chars = whitespace_check(first_line, c);
                    if (ws_chars == 0) {
                        auto msg = "Text block's first line must start with whitespace.";
                        throw StaticError(filename, begin, msg);
                    }
                    while (true) {
                        assert(ws_chars > 0);
                        // Read up to the \n
                        for (c = &c[ws_chars]; *c != '\n' ; ++c) {
                            if (*c == '\0')
                                throw StaticError(filename, begin, "Unexpected EOF");
                            block << *c;
                        }
                        // Add the \n
                        block << '\n';
                        ++c;
                        line_number++;
                        line_start = c;
                        // Examine next line
                        ws_chars = whitespace_check(first_line, c);
                        if (ws_chars == 0) {
                            // End of text block
                            // Skip over any whitespace
                            while (*c == ' ' || *c == '\t') ++c;
                            // Expect |||
                            if (!(*c == '|' && *(c+1) == '|' && *(c+2) == '|')) {
                                auto msg = "Text block not terminated with |||";
                                throw StaticError(filename, begin, msg);
                            }
                            c += 2;  // Leave on the last |
                            data = block.str();
                            kind = Token::STRING;
                            break;
                        }
                    }

                    break;  // Out of the switch.
                }

                for (; *c != '\0' ; ++c) {
                    if (!is_symbol(*c)) {
                        break;
                    }
                    data += *c;
                }
                --c;
                kind = Token::OPERATOR;
            } else {
                std::stringstream ss;
                ss << "Could not lex the character ";
                auto uc = (unsigned char)(*c);
                if (*c < 32)
                    ss << "code " << unsigned(uc);
                else
                    ss << "'" << *c << "'";
                throw StaticError(filename, begin, ss.str());
            }
            break;
        }

        Location end(line_number, c - line_start + 1);
        r.push_back(Token(kind, data, LocationRange(filename, begin, end)));
    }

    Location end(line_number, c - line_start + 1);
    r.push_back(Token(Token::END_OF_FILE, "", LocationRange(filename, end, end)));
    return r;
}
Example #5
0
static int llex(LexState *ls, TValue *tv)
{
  lj_str_resetbuf(&ls->sb);
  for (;;) {
    if (lj_char_isident(ls->current)) {
      GCstr *s;
      if (lj_char_isdigit(ls->current)) {  /* Numeric literal. */
	lex_number(ls, tv);
	return TK_number;
      }
      /* Identifier or reserved word. */
      do {
	save_and_next(ls);
      } while (lj_char_isident(ls->current));
      s = lj_parse_keepstr(ls, ls->sb.buf, ls->sb.n);
      setstrV(ls->L, tv, s);
      if (s->reserved > 0)  /* Reserved word? */
	return TK_OFS + s->reserved;
      return TK_name;
    }
    switch (ls->current) {
    case '\n':
    case '\r':
      inclinenumber(ls);
      continue;
    case ' ':
    case '\t':
    case '\v':
    case '\f':
      next(ls);
      continue;
    case '-':
      next(ls);
      if (ls->current != '-') return '-';
      /* else is a comment */
      next(ls);
      if (ls->current == '[') {
	int sep = skip_sep(ls);
	lj_str_resetbuf(&ls->sb);  /* `skip_sep' may dirty the buffer */
	if (sep >= 0) {
	  read_long_string(ls, NULL, sep);  /* long comment */
	  lj_str_resetbuf(&ls->sb);
	  continue;
	}
      }
      /* else short comment */
      while (!currIsNewline(ls) && ls->current != END_OF_STREAM)
	next(ls);
      continue;
    case '[': {
      int sep = skip_sep(ls);
      if (sep >= 0) {
	read_long_string(ls, tv, sep);
	return TK_string;
      } else if (sep == -1) {
	return '[';
      } else {
	lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
	continue;
      }
      }
    case '=':
      next(ls);
      if (ls->current != '=') return '='; else { next(ls); return TK_eq; }
    case '<':
      next(ls);
      if (ls->current != '=') return '<'; else { next(ls); return TK_le; }
    case '>':
      next(ls);
      if (ls->current != '=') return '>'; else { next(ls); return TK_ge; }
    case '~':
      next(ls);
      if (ls->current != '=') return '~'; else { next(ls); return TK_ne; }
    case ':':
      next(ls);
      if (ls->current != ':') return ':'; else { next(ls); return TK_label; }
    case '"':
    case '\'':
      read_string(ls, ls->current, tv);
      return TK_string;
    case '.':
      save_and_next(ls);
      if (ls->current == '.') {
	next(ls);
	if (ls->current == '.') {
	  next(ls);
	  return TK_dots;   /* ... */
	}
	return TK_concat;   /* .. */
      } else if (!lj_char_isdigit(ls->current)) {
	return '.';
      } else {
	lex_number(ls, tv);
	return TK_number;
      }
    case END_OF_STREAM:
      return TK_eof;
    default: {
      int c = ls->current;
      next(ls);
      return c;  /* Single-char tokens (+ - / ...). */
    }
    }
  }
}
Example #6
0
/* Lex a token into pfile->cur_token, which is also incremented, to
   get diagnostics pointing to the correct location.

   Does not handle issues such as token lookahead, multiple-include
   optimization, directives, skipping etc.  This function is only
   suitable for use by _cpp_lex_token, and in special cases like
   lex_expansion_token which doesn't care for any of these issues.

   When meeting a newline, returns CPP_EOF if parsing a directive,
   otherwise returns to the start of the token buffer if permissible.
   Returns the location of the lexed token.  */
cpp_token *
_cpp_lex_direct (cpp_reader *pfile)
{
  cppchar_t c;
  cpp_buffer *buffer;
  const unsigned char *comment_start;
  cpp_token *result = pfile->cur_token++;

 fresh_line:
  result->flags = 0;
  buffer = pfile->buffer;
  if (buffer->need_line)
    {
      if (!_cpp_get_fresh_line (pfile))
	{
	  result->type = CPP_EOF;
	  if (!pfile->state.in_directive)
	    {
	      /* Tell the compiler the line number of the EOF token.  */
	      result->line = pfile->line;
	      result->flags = BOL;
	    }
	  return result;
	}
      if (!pfile->keep_tokens)
	{
	  pfile->cur_run = &pfile->base_run;
	  result = pfile->base_run.base;
	  pfile->cur_token = result + 1;
	}
      result->flags = BOL;
      if (pfile->state.parsing_args == 2)
	result->flags |= PREV_WHITE;
    }
  buffer = pfile->buffer;
 update_tokens_line:
  result->line = pfile->line;

 skipped_white:
  if (buffer->cur >= buffer->notes[buffer->cur_note].pos
      && !pfile->overlaid_buffer)
    {
      _cpp_process_line_notes (pfile, false);
      result->line = pfile->line;
    }
  c = *buffer->cur++;
  result->col = CPP_BUF_COLUMN (buffer, buffer->cur);

  switch (c)
    {
    case ' ': case '\t': case '\f': case '\v': case '\0':
      result->flags |= PREV_WHITE;
      skip_whitespace (pfile, c);
      goto skipped_white;

    case '\n':
      pfile->line++;
      buffer->need_line = true;
      goto fresh_line;

    case '0': case '1': case '2': case '3': case '4':
    case '5': case '6': case '7': case '8': case '9':
      result->type = CPP_NUMBER;
      lex_number (pfile, &result->val.str);
      break;

    case 'L':
      /* 'L' may introduce wide characters or strings.  */
      if (*buffer->cur == '\'' || *buffer->cur == '"')
	{
	  lex_string (pfile, result, buffer->cur - 1);
	  break;
	}
      /* Fall through.  */

    case '_':
    case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
    case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
    case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
    case 's': case 't': case 'u': case 'v': case 'w': case 'x':
    case 'y': case 'z':
    case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
    case 'G': case 'H': case 'I': case 'J': case 'K':
    case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
    case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
    case 'Y': case 'Z':
      result->type = CPP_NAME;
      result->val.node = lex_identifier (pfile, buffer->cur - 1);

      /* Convert named operators to their proper types.  */
      if (result->val.node->flags & NODE_OPERATOR)
	{
	  result->flags |= NAMED_OP;
	  result->type = result->val.node->directive_index;
	}
      break;

    case '\'':
    case '"':
      lex_string (pfile, result, buffer->cur - 1);
      break;

    case '/':
      /* A potential block or line comment.  */
      comment_start = buffer->cur;
      c = *buffer->cur;
      
      if (c == '*')
	{
	  if (_cpp_skip_block_comment (pfile))
	    cpp_error (pfile, CPP_DL_ERROR, "unterminated comment");
	}
      else if (c == '/' && (CPP_OPTION (pfile, cplusplus_comments)
			    || CPP_IN_SYSTEM_HEADER (pfile)))
	{
	  /* Warn about comments only if pedantically GNUC89, and not
	     in system headers.  */
	  if (CPP_OPTION (pfile, lang) == CLK_GNUC89 && CPP_PEDANTIC (pfile)
	      && ! buffer->warned_cplusplus_comments)
	    {
	      cpp_error (pfile, CPP_DL_PEDWARN,
			 "C++ style comments are not allowed in ISO C90");
	      cpp_error (pfile, CPP_DL_PEDWARN,
			 "(this will be reported only once per input file)");
	      buffer->warned_cplusplus_comments = 1;
	    }

	  if (skip_line_comment (pfile) && CPP_OPTION (pfile, warn_comments))
	    cpp_error (pfile, CPP_DL_WARNING, "multi-line comment");
	}
      else if (c == '=')
	{
	  buffer->cur++;
	  result->type = CPP_DIV_EQ;
	  break;
	}
      else
	{
	  result->type = CPP_DIV;
	  break;
	}

      if (!pfile->state.save_comments)
	{
	  result->flags |= PREV_WHITE;
	  goto update_tokens_line;
	}

      /* Save the comment as a token in its own right.  */
      save_comment (pfile, result, comment_start, c);
      break;

    case '<':
      if (pfile->state.angled_headers)
	{
	  lex_string (pfile, result, buffer->cur - 1);
	  break;
	}

      result->type = CPP_LESS;
      if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_LESS_EQ;
      else if (*buffer->cur == '<')
	{
	  buffer->cur++;
	  IF_NEXT_IS ('=', CPP_LSHIFT_EQ, CPP_LSHIFT);
	}
      else if (*buffer->cur == '?' && CPP_OPTION (pfile, cplusplus))
	{
	  buffer->cur++;
	  IF_NEXT_IS ('=', CPP_MIN_EQ, CPP_MIN);
	}
      else if (CPP_OPTION (pfile, digraphs))
	{
	  if (*buffer->cur == ':')
	    {
	      buffer->cur++;
	      result->flags |= DIGRAPH;
	      result->type = CPP_OPEN_SQUARE;
	    }
	  else if (*buffer->cur == '%')
	    {
	      buffer->cur++;
	      result->flags |= DIGRAPH;
	      result->type = CPP_OPEN_BRACE;
	    }
	}
      break;

    case '>':
      result->type = CPP_GREATER;
      if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_GREATER_EQ;
      else if (*buffer->cur == '>')
	{
	  buffer->cur++;
	  IF_NEXT_IS ('=', CPP_RSHIFT_EQ, CPP_RSHIFT);
	}
      else if (*buffer->cur == '?' && CPP_OPTION (pfile, cplusplus))
	{
	  buffer->cur++;
	  IF_NEXT_IS ('=', CPP_MAX_EQ, CPP_MAX);
	}
      break;

    case '%':
      result->type = CPP_MOD;
      if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_MOD_EQ;
      else if (CPP_OPTION (pfile, digraphs))
	{
	  if (*buffer->cur == ':')
	    {
	      buffer->cur++;
	      result->flags |= DIGRAPH;
	      result->type = CPP_HASH;
	      if (*buffer->cur == '%' && buffer->cur[1] == ':')
		buffer->cur += 2, result->type = CPP_PASTE;
	    }
	  else if (*buffer->cur == '>')
	    {
	      buffer->cur++;
	      result->flags |= DIGRAPH;
	      result->type = CPP_CLOSE_BRACE;
	    }
	}
      break;

    case '.':
      result->type = CPP_DOT;
      if (ISDIGIT (*buffer->cur))
	{
	  result->type = CPP_NUMBER;
	  lex_number (pfile, &result->val.str);
	}
      else if (*buffer->cur == '.' && buffer->cur[1] == '.')
	buffer->cur += 2, result->type = CPP_ELLIPSIS;
      else if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
	buffer->cur++, result->type = CPP_DOT_STAR;
      break;

    case '+':
      result->type = CPP_PLUS;
      if (*buffer->cur == '+')
	buffer->cur++, result->type = CPP_PLUS_PLUS;
      else if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_PLUS_EQ;
      break;

    case '-':
      result->type = CPP_MINUS;
      if (*buffer->cur == '>')
	{
	  buffer->cur++;
	  result->type = CPP_DEREF;
	  if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
	    buffer->cur++, result->type = CPP_DEREF_STAR;
	}
      else if (*buffer->cur == '-')
	buffer->cur++, result->type = CPP_MINUS_MINUS;
      else if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_MINUS_EQ;
      break;

    case '&':
      result->type = CPP_AND;
      if (*buffer->cur == '&')
	buffer->cur++, result->type = CPP_AND_AND;
      else if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_AND_EQ;
      break;

    case '|':
      result->type = CPP_OR;
      if (*buffer->cur == '|')
	buffer->cur++, result->type = CPP_OR_OR;
      else if (*buffer->cur == '=')
	buffer->cur++, result->type = CPP_OR_EQ;
      break;

    case ':':
      result->type = CPP_COLON;
      if (*buffer->cur == ':' && CPP_OPTION (pfile, cplusplus))
	buffer->cur++, result->type = CPP_SCOPE;
      else if (*buffer->cur == '>' && CPP_OPTION (pfile, digraphs))
	{
	  buffer->cur++;
	  result->flags |= DIGRAPH;
	  result->type = CPP_CLOSE_SQUARE;
	}
      break;

    case '*': IF_NEXT_IS ('=', CPP_MULT_EQ, CPP_MULT); break;
    case '=': IF_NEXT_IS ('=', CPP_EQ_EQ, CPP_EQ); break;
    case '!': IF_NEXT_IS ('=', CPP_NOT_EQ, CPP_NOT); break;
    case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break;
    case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); break;

    case '?': result->type = CPP_QUERY; break;
    case '~': result->type = CPP_COMPL; break;
    case ',': result->type = CPP_COMMA; break;
    case '(': result->type = CPP_OPEN_PAREN; break;
    case ')': result->type = CPP_CLOSE_PAREN; break;
    case '[': result->type = CPP_OPEN_SQUARE; break;
    case ']': result->type = CPP_CLOSE_SQUARE; break;
    case '{': result->type = CPP_OPEN_BRACE; break;
    case '}': result->type = CPP_CLOSE_BRACE; break;
    case ';': result->type = CPP_SEMICOLON; break;

      /* @ is a punctuator in Objective-C.  */
    case '@': result->type = CPP_ATSIGN; break;

    case '$':
    case '\\':
      {
	const uchar *base = --buffer->cur;

	if (forms_identifier_p (pfile, true))
	  {
	    result->type = CPP_NAME;
	    result->val.node = lex_identifier (pfile, base);
	    break;
	  }
	buffer->cur++;
      }

    default:
      create_literal (pfile, result, buffer->cur - 1, 1, CPP_OTHER);
      break;
    }

  return result;
}
Example #7
0
pdf_token
pdf_lex(fz_stream *f, pdf_lexbuf *buf)
{
	while (1)
	{
		int c = fz_read_byte(f);
		switch (c)
		{
		case EOF:
			return PDF_TOK_EOF;
		case IS_WHITE:
			lex_white(f);
			break;
		case '%':
			lex_comment(f);
			break;
		case '/':
			lex_name(f, buf);
			return PDF_TOK_NAME;
		case '(':
			return lex_string(f, buf);
		case ')':
			fz_warn(f->ctx, "lexical error (unexpected ')')");
			continue;
		case '<':
			c = fz_read_byte(f);
			if (c == '<')
			{
				return PDF_TOK_OPEN_DICT;
			}
			else
			{
				fz_unread_byte(f);
				return lex_hex_string(f, buf);
			}
		case '>':
			c = fz_read_byte(f);
			if (c == '>')
			{
				return PDF_TOK_CLOSE_DICT;
			}
			fz_warn(f->ctx, "lexical error (unexpected '>')");
			if (c == EOF)
			{
				return PDF_TOK_EOF;
			}
			fz_unread_byte(f);
			continue;
		case '[':
			return PDF_TOK_OPEN_ARRAY;
		case ']':
			return PDF_TOK_CLOSE_ARRAY;
		case '{':
			return PDF_TOK_OPEN_BRACE;
		case '}':
			return PDF_TOK_CLOSE_BRACE;
		case IS_NUMBER:
			/* cf. https://code.google.com/p/sumatrapdf/issues/detail?id=2231 */
			{
				int tok = lex_number(f, buf, c);
				while (1)
				{
					c = fz_peek_byte(f);
					switch (c)
					{
					case IS_NUMBER:
						fz_warn(f->ctx, "ignoring invalid character after number: '%c'", c);
						fz_read_byte(f);
						continue;
					default:
						return tok;
					}
				}
			}
		default: /* isregular: !isdelim && !iswhite && c != EOF */
			fz_unread_byte(f);
			lex_name(f, buf);
			return pdf_token_from_keyword(buf->scratch);
		}
	}
}
int yylex(void)
{
    int ch;

    for(;;) {
  ch = getchar();
  switch( ch ) {
    case EOF:
      return 0;
    case '\n':
      ++yylineno;
      continue;
    case '\t': case ' ':
      continue;
    case '{':
      do {
    ch = getchar();
      } while( ch != '}' && ch != EOF );
      continue;
    case ':':  /* ASSIGN or COLON */
      ch = getchar();
      switch( ch ) {
        case '=':
    return lex_log_token(T_ASSIGN);
        default:
    ungetc(ch, stdin);
    return lex_log_token(T_COLON);
      }
    case ',':
      return lex_log_token(T_COMMA);
    case '.':  /* RCON or DOT */
      ch = getchar();
      ungetc(ch, stdin);
      if( isdigit(ch) )
    return lex_log_token(lex_number('.'));
      else
    return lex_log_token(T_DOT);
    case '[':
      return lex_log_token(T_LBRACK);
    case ']':
      return lex_log_token(T_RBRACK);
    case '(':
      return lex_log_token(T_LPAREN);
    case ')':
      return lex_log_token(T_RPAREN);
    case '<':  /* NE, LE, or LT */
      ch = getchar();
      switch( ch ) {
        case '>':
    return lex_log_token(T_NE);
        case '=':
    return lex_log_token(T_LE);
        default:
    ungetc(ch, stdin);
    return lex_log_token(T_LT);
      }
    case '=':
      return lex_log_token(T_EQ);
    case '>':  /* GE or GT */
      ch = getchar();
      switch( ch ) {
        case '=':
    return lex_log_token(T_GE);
        default:
    ungetc(ch, stdin);
    return lex_log_token(T_GT);
      }
    case '-':
      return lex_log_token(T_MINUS);
    case '*':
      return lex_log_token(T_MUL);
    case '+':
      return lex_log_token(T_PLUS);
    case '/':
      return lex_log_token(T_RDIV);
    case ';':
      return lex_log_token(T_SEMI);
    case '&':
      return lex_log_token(T_AMPER);
    case '^':
      return lex_log_token(T_CARET);
    default:
      if( isalpha(ch) || ch == '_' )
    return lex_log_token(lex_alpha_ident(ch));
      else if( isdigit(ch) )
    return lex_log_token(lex_number(ch));
      else
    lexerror("Illegal character %03o", ch);
  }
    }
}
Example #9
0
int
cmd_t_test (struct lexer *lexer, struct dataset *ds)
{
  bool ok;
  const struct dictionary *dict = dataset_dict (ds);
  struct tt tt;
  int mode_count = 0;

  /* Variables pertaining to the paired mode */
  const struct variable **v1 = NULL;
  size_t n_v1;
  const struct variable **v2 = NULL;
  size_t n_v2;
	  
  size_t n_pairs = 0;
  vp *pairs = NULL;


  /* One sample mode */
  double testval = SYSMIS;

  /* Independent samples mode */
  const struct variable *gvar;
  union value gval0;
  union value gval1;
  bool cut = false;

  tt.wv = dict_get_weight (dict);
  tt.dict = dict;
  tt.confidence = 0.95;
  tt.exclude = MV_ANY;
  tt.missing_type = MISS_ANALYSIS;
  tt.n_vars = 0;
  tt.vars = NULL;
  tt.mode = MODE_undef;

  lex_match (lexer, T_EQUALS);

  for (; lex_token (lexer) != T_ENDCMD; )
    {
      lex_match (lexer, T_SLASH);
      if (lex_match_id (lexer, "TESTVAL"))
	{
	  mode_count++;
	  tt.mode = MODE_SINGLE;
	  lex_match (lexer, T_EQUALS);
	  lex_force_num (lexer);
	  testval = lex_number (lexer);
	  lex_get (lexer);
	}
      else if (lex_match_id (lexer, "GROUPS"))
	{
	  mode_count++;
	  cut = false;
	  tt.mode = MODE_INDEP;
	  lex_match (lexer, T_EQUALS);

	  if (NULL == (gvar = parse_variable (lexer, dict)))
	    goto parse_failed;
      
	  if (lex_match (lexer, T_LPAREN))
	    {

	      value_init (&gval0, var_get_width (gvar));
	      parse_value (lexer, &gval0, gvar);
	      cut = true;
	      if (lex_match (lexer, T_COMMA))
		{
		  value_init (&gval1, var_get_width (gvar));
		  parse_value (lexer, &gval1, gvar);
		  cut = false;
		}

	      lex_force_match (lexer, T_RPAREN);
	    }
	  else
	    {
	      value_init (&gval0, 0);
	      value_init (&gval1, 0);
	      gval0.f = 1.0;
	      gval1.f = 2.0;
	      cut = false;
	    }

	  if ( cut == true && var_is_alpha (gvar))
	    {
	      msg (SE, _("When applying GROUPS to a string variable, two "
			 "values must be specified."));
	      goto parse_failed;
	    }
	}
      else if (lex_match_id (lexer, "PAIRS"))
	{
	  bool with = false;
	  bool paired = false;

	  if (tt.n_vars > 0)
	    {
	      msg (SE, _("VARIABLES subcommand may not be used with PAIRS."));
	      goto parse_failed;
	    }

	  mode_count++;
	  tt.mode = MODE_PAIRED;
	  lex_match (lexer, T_EQUALS);

	  if (!parse_variables_const (lexer, dict,
				      &v1, &n_v1,
				      PV_NO_DUPLICATE | PV_NUMERIC))
	    goto parse_failed;

	  if ( lex_match (lexer, T_WITH))
	    {
	      with = true;
	      if (!parse_variables_const (lexer, dict,
					  &v2, &n_v2,
					  PV_NO_DUPLICATE | PV_NUMERIC))
		goto parse_failed;

	      if (lex_match (lexer, T_LPAREN)
		  && lex_match_id (lexer, "PAIRED")
		  && lex_match (lexer, T_RPAREN))
		{
		  paired = true;
		  if (n_v1 != n_v2)
		    {
		      msg (SE, _("PAIRED was specified but the number of variables "
				 "preceding WITH (%zu) did not match the number "
				 "following (%zu)."),
			   n_v1, n_v2);
		      goto parse_failed;
		    }
		}
	    }
	  {
	    int i;

	    if ( !with )
	      n_pairs = (n_v1 * (n_v1 - 1)) / 2.0;
	    else if ( paired )
	      n_pairs = n_v1;
	    else
	      n_pairs = n_v1 * n_v2;
	  
	    pairs = xcalloc (n_pairs, sizeof *pairs);

	    if ( with)
	      {
		int x = 0;
		if (paired)
		  {
		    for (i = 0 ; i < n_v1; ++i)
		      {
			vp *pair = &pairs[i];
			(*pair)[0] = v1[i];
			(*pair)[1] = v2[i];
		      }	
		  }
		else
		  {
		    for (i = 0 ; i < n_v1; ++i)
		      {
			int j;
			for (j = 0 ; j < n_v2; ++j)
			  {
			    vp *pair = &pairs[x++];
			    (*pair)[0] = v1[i];
			    (*pair)[1] = v2[j];
			  }
		      }
		  }
	      }
	    else
	      {
		int x = 0;
		for (i = 0 ; i < n_v1; ++i)
		  {
		    int j;

		    for (j = i + 1 ; j < n_v1; ++j)
		      {
			vp *pair = &pairs[x++];
			(*pair)[0] = v1[i];
			(*pair)[1] = v1[j];
		      }
		  }
	      }

	  }
	}
      else if (lex_match_id (lexer, "VARIABLES"))
	{
	  if ( tt.mode == MODE_PAIRED)
	    {
	      msg (SE, _("VARIABLES subcommand may not be used with PAIRS."));
	      goto parse_failed;
	    }

	  lex_match (lexer, T_EQUALS);

	  if (!parse_variables_const (lexer, dict,
				      &tt.vars,
				      &tt.n_vars,
				      PV_NO_DUPLICATE | PV_NUMERIC))
	    goto parse_failed;
	}
      else if ( lex_match_id (lexer, "MISSING"))
	{
          lex_match (lexer, T_EQUALS);
          while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
            {
	      if (lex_match_id (lexer, "INCLUDE"))
		{
		  tt.exclude = MV_SYSTEM;
		}
	      else if (lex_match_id (lexer, "EXCLUDE"))
		{
		  tt.exclude = MV_ANY;
		}
	      else if (lex_match_id (lexer, "LISTWISE"))
		{
		  tt.missing_type = MISS_LISTWISE;
		}
	      else if (lex_match_id (lexer, "ANALYSIS"))
		{
		  tt.missing_type = MISS_ANALYSIS;
		}
	      else
		{
                  lex_error (lexer, NULL);
		  goto parse_failed;
		}
	      lex_match (lexer, T_COMMA);
	    }
	}
      else if (lex_match_id (lexer, "CRITERIA"))
	{
          lex_match (lexer, T_EQUALS);
	  if ( lex_force_match_id (lexer, "CIN"))
	    if ( lex_force_match (lexer, T_LPAREN))
	      {
		lex_force_num (lexer);
		tt.confidence = lex_number (lexer);
		lex_get (lexer);
		lex_force_match (lexer, T_RPAREN);
	      }
	}
      else 
	{
	  lex_error (lexer, NULL);
	  goto parse_failed;
	}
    }

  if ( mode_count != 1)
    {
      msg (SE, _("Exactly one of TESTVAL, GROUPS and PAIRS subcommands "
		 "must be specified."));
      goto parse_failed;
    }

  if (tt.n_vars == 0 && tt.mode != MODE_PAIRED)
    {
      lex_sbc_missing ("VARIABLES");
      goto parse_failed;
    }



  /* Deal with splits etc */
  {
    struct casereader *group;
    struct casegrouper *grouper = casegrouper_create_splits (proc_open (ds), dict);

    while (casegrouper_get_next_group (grouper, &group))
      {
	if ( tt.mode == MODE_SINGLE)
	  {
	    if ( tt.missing_type == MISS_LISTWISE )
	      group  = casereader_create_filter_missing (group,
							 tt.vars, tt.n_vars,
							 tt.exclude,
							 NULL,  NULL);
	    one_sample_run (&tt, testval, group);
	  }
	else if ( tt.mode == MODE_PAIRED)
	  {
	    if ( tt.missing_type == MISS_LISTWISE )
	      {
		group  = casereader_create_filter_missing (group,
							   v1, n_v1,
							   tt.exclude,
							   NULL,  NULL);
		group  = casereader_create_filter_missing (group,
							   v2, n_v2,
							   tt.exclude,
							   NULL,  NULL);
	      }

	    paired_run (&tt, n_pairs, pairs, group);
	  }
	else /* tt.mode == MODE_INDEP */
	  {
	    if ( tt.missing_type == MISS_LISTWISE )
	      {
		group  = casereader_create_filter_missing (group,
							   tt.vars, tt.n_vars,
							   tt.exclude,
							   NULL,  NULL);

		group  = casereader_create_filter_missing (group,
							   &gvar, 1,
							   tt.exclude,
							   NULL,  NULL);

	      }

	    indep_run (&tt, gvar, cut, &gval0, &gval1, group);
	  }
      }

    ok = casegrouper_destroy (grouper);
    ok = proc_commit (ds) && ok;
  }

  free (pairs);
  free (v1);
  free (v2);

  free (tt.vars);

  return ok ? CMD_SUCCESS : CMD_FAILURE;

 parse_failed:
  return CMD_FAILURE;
}
Example #10
0
bool Lexer::lex_file(char * filename) {
    FILE * fp = fopen(filename, "r");
    if (fp == NULL) {
        REPORT_ERROR("Failed to open file [%s].", filename);
        return false;
    }

    current_filename = filename;

    char cur;
    while (true) {
        cur = peek_char(fp);
		while (cur && isspace(cur)) {
            next_char(fp);
			cur = peek_char(fp);
        }
        if (!cur) {
            break;
        }

        if (isalpha(cur)) {
            if (!lex_identifier(fp)) {    
                fclose(fp);
                return false;
            }
            continue;
        }
        else if (isdigit(cur)) {
            if (!lex_number(fp)) {
                fclose(fp);
                return false;
            }
            continue;
        }
        else if (cur == '\'') {
            if (!lex_char_literal(fp)) {
                fclose(fp);
                return false;
            }
            continue;
        }
        else if (cur == '\"') {
            if (!lex_string_literal(fp)) {
                fclose(fp);
                return false;
            }
            continue;
        }
		else if (cur == '\0') {
			assert(0);
		}
        else {
            if (!lex_symbol(fp)) {
                fclose(fp);
                return false;
            }
            continue;
        }
    }

    fclose(fp);
    return true;
}
Example #11
0
rev_list *
rev_list_cvs (cvs_file *cvs)
{
    rev_list	*rl = calloc (1, sizeof (rev_list));
    cvs_number	trunk_number;
    rev_commit	*trunk; 
    rev_commit	*branch;
    cvs_version	*cv;
    cvs_branch	*cb;
    rev_ref	*t;
    cvs_version	*ctrunk = NULL;

    build_branches();
    /*
     * Locate first revision on trunk branch
     */
    for (cv = cvs->versions; cv; cv = cv->next) {
	if (cvs_is_trunk (&cv->number) &&
	    (!ctrunk || cvs_number_compare (&cv->number,
					    &ctrunk->number) < 0))
	{
	    ctrunk = cv;
	}
    }
    /*
     * Generate trunk branch
     */
    if (ctrunk)
	trunk_number = ctrunk->number;
    else
	trunk_number = lex_number ("1.1");
    trunk = rev_branch_cvs (cvs, &trunk_number);
    if (trunk) {
	t = rev_list_add_head (rl, trunk, atom ("master"), 2);
	t->number = trunk_number;
    }
    else
	fprintf(stderr, "warning - no master branch generated\n");
    /*
     * Search for other branches
     */
#if DEBUG
    printf ("building branches for %s\n", cvs->name);
#endif
    
    for (cv = cvs->versions; cv; cv = cv->next) {
	for (cb = cv->branches; cb; cb = cb->next)
	{
	    branch = rev_branch_cvs (cvs, &cb->number);
	    rev_list_add_head (rl, branch, NULL, 0);
	}
    }
    rev_list_patch_vendor_branch (rl, cvs);
    rev_list_graft_branches (rl, cvs);
    rev_list_set_refs (rl, cvs);
    rev_list_sort_heads (rl, cvs);
    rev_list_set_tail (rl);
    rev_list_free_dead_files (rl);
    rev_list_validate (rl);
    return rl;
}
Example #12
0
fz_error
pdf_lex(int *tok, fz_stream *f, char *buf, int n, int *sl)
{
	while (1)
	{
		int c = fz_read_byte(f);
		switch (c)
		{
		case EOF:
			*tok = PDF_TOK_EOF;
			return fz_okay;
		case IS_WHITE:
			lex_white(f);
			break;
		case '%':
			lex_comment(f);
			break;
		case '/':
			lex_name(f, buf, n);
			*sl = strlen(buf);
			*tok = PDF_TOK_NAME;
			return fz_okay;
		case '(':
			*sl = lex_string(f, buf, n);
			*tok = PDF_TOK_STRING;
			return fz_okay;
		case ')':
			*tok = PDF_TOK_ERROR;
			goto cleanuperror;
		case '<':
			c = fz_read_byte(f);
			if (c == '<')
			{
				*tok = PDF_TOK_OPEN_DICT;
			}
			else
			{
				fz_unread_byte(f);
				*sl = lex_hex_string(f, buf, n);
				*tok = PDF_TOK_STRING;
			}
			return fz_okay;
		case '>':
			c = fz_read_byte(f);
			if (c == '>')
			{
				*tok = PDF_TOK_CLOSE_DICT;
				return fz_okay;
			}
			*tok = PDF_TOK_ERROR;
			goto cleanuperror;
		case '[':
			*tok = PDF_TOK_OPEN_ARRAY;
			return fz_okay;
		case ']':
			*tok = PDF_TOK_CLOSE_ARRAY;
			return fz_okay;
		case '{':
			*tok = PDF_TOK_OPEN_BRACE;
			return fz_okay;
		case '}':
			*tok = PDF_TOK_CLOSE_BRACE;
			return fz_okay;
		case IS_NUMBER:
			fz_unread_byte(f);
			*sl = lex_number(f, buf, n, tok);
			return fz_okay;
		default: /* isregular: !isdelim && !iswhite && c != EOF */
			fz_unread_byte(f);
			lex_name(f, buf, n);
			*sl = strlen(buf);
			*tok = pdf_token_from_keyword(buf);
			return fz_okay;
		}
	}

cleanuperror:
	*tok = PDF_TOK_ERROR;
	return fz_throw("lexical error");
}
Example #13
0
int
cmd_reliability (struct lexer *lexer, struct dataset *ds)
{
  const struct dictionary *dict = dataset_dict (ds);

  struct reliability reliability;
  reliability.n_variables = 0;
  reliability.variables = NULL;
  reliability.model = MODEL_ALPHA;
    reliability.exclude = MV_ANY;
  reliability.summary = 0;

  reliability.wv = dict_get_weight (dict);

  reliability.total_start = 0;

  lex_match (lexer, T_SLASH);

  if (!lex_force_match_id (lexer, "VARIABLES"))
    {
      goto error;
    }

  lex_match (lexer, T_EQUALS);

  if (!parse_variables_const (lexer, dict, &reliability.variables, &reliability.n_variables,
			      PV_NO_DUPLICATE | PV_NUMERIC))
    goto error;

  if (reliability.n_variables < 2)
    msg (MW, _("Reliability on a single variable is not useful."));


    {
      int i;
      struct cronbach *c;
      /* Create a default Scale */

      reliability.n_sc = 1;
      reliability.sc = xzalloc (sizeof (struct cronbach) * reliability.n_sc);

      ds_init_cstr (&reliability.scale_name, "ANY");

      c = &reliability.sc[0];
      c->n_items = reliability.n_variables;
      c->items = xzalloc (sizeof (struct variable*) * c->n_items);

      for (i = 0 ; i < c->n_items ; ++i)
	c->items[i] = reliability.variables[i];
    }



  while (lex_token (lexer) != T_ENDCMD)
    {
      lex_match (lexer, T_SLASH);

      if (lex_match_id (lexer, "SCALE"))
	{
	  struct const_var_set *vs;
	  if ( ! lex_force_match (lexer, T_LPAREN))
	    goto error;

	  if ( ! lex_force_string (lexer) ) 
	    goto error;

	  ds_init_substring (&reliability.scale_name, lex_tokss (lexer));

	  lex_get (lexer);

	  if ( ! lex_force_match (lexer, T_RPAREN))
	    goto error;

          lex_match (lexer, T_EQUALS);

	  vs = const_var_set_create_from_array (reliability.variables, reliability.n_variables);


	  if (!parse_const_var_set_vars (lexer, vs, &reliability.sc->items, &reliability.sc->n_items, 0))
	    {
	      const_var_set_destroy (vs);
	      goto error;
	    }

	  const_var_set_destroy (vs);
	}
      else if (lex_match_id (lexer, "MODEL"))
	{
          lex_match (lexer, T_EQUALS);
	  if (lex_match_id (lexer, "ALPHA"))
	    {
	      reliability.model = MODEL_ALPHA;
	    }
	  else if (lex_match_id (lexer, "SPLIT"))
	    {
	      reliability.model = MODEL_SPLIT;
	      reliability.split_point = -1;

	      if ( lex_match (lexer, T_LPAREN))
		{
		  lex_force_num (lexer);
		  reliability.split_point = lex_number (lexer);
		  lex_get (lexer);
		  lex_force_match (lexer, T_RPAREN);
		}
	    }
	  else
	    goto error;
	}
      else if (lex_match_id (lexer, "SUMMARY"))
        {
          lex_match (lexer, T_EQUALS);
	  if (lex_match_id (lexer, "TOTAL"))
	    {
	      reliability.summary |= SUMMARY_TOTAL;
	    }
	  else if (lex_match (lexer, T_ALL))
	    {
	      reliability.summary = 0xFFFF;
	    }
	  else
	    goto error;
	}
      else if (lex_match_id (lexer, "MISSING"))
        {
          lex_match (lexer, T_EQUALS);
          while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
            {
	      if (lex_match_id (lexer, "INCLUDE"))
		{
		  reliability.exclude = MV_SYSTEM;
		}
	      else if (lex_match_id (lexer, "EXCLUDE"))
		{
		  reliability.exclude = MV_ANY;
		}
	      else
		{
                  lex_error (lexer, NULL);
		  goto error;
		}
	    }
	}
      else
	{
	  lex_error (lexer, NULL);
	  goto error;
	}
    }

  if ( reliability.model == MODEL_SPLIT)
    {
      int i;
      const struct cronbach *s;

      if ( reliability.split_point >= reliability.n_variables)
        {
          msg (ME, _("The split point must be less than the number of variables"));
          goto error;
        }

      reliability.n_sc += 2 ;
      reliability.sc = xrealloc (reliability.sc, sizeof (struct cronbach) * reliability.n_sc);

      s = &reliability.sc[0];

      reliability.sc[1].n_items =
	(reliability.split_point == -1) ? s->n_items / 2 : reliability.split_point;

      reliability.sc[2].n_items = s->n_items - reliability.sc[1].n_items;
      reliability.sc[1].items = xzalloc (sizeof (struct variable *)
				 * reliability.sc[1].n_items);

      reliability.sc[2].items = xzalloc (sizeof (struct variable *) *
				 reliability.sc[2].n_items);

      for  (i = 0; i < reliability.sc[1].n_items ; ++i)
	reliability.sc[1].items[i] = s->items[i];

      while (i < s->n_items)
	{
	  reliability.sc[2].items[i - reliability.sc[1].n_items] = s->items[i];
	  i++;
	}
    }

  if ( reliability.summary & SUMMARY_TOTAL)
    {
      int i;
      const int base_sc = reliability.n_sc;

      reliability.total_start = base_sc;

      reliability.n_sc +=  reliability.sc[0].n_items ;
      reliability.sc = xrealloc (reliability.sc, sizeof (struct cronbach) * reliability.n_sc);


      for (i = 0 ; i < reliability.sc[0].n_items; ++i )
	{
	  int v_src;
	  int v_dest = 0;
	  struct cronbach *s = &reliability.sc[i + base_sc];

	  s->n_items = reliability.sc[0].n_items - 1;
	  s->items = xzalloc (sizeof (struct variable *) * s->n_items);
	  for (v_src = 0 ; v_src < reliability.sc[0].n_items ; ++v_src)
	    {
	      if ( v_src != i)
		s->items[v_dest++] = reliability.sc[0].items[v_src];
	    }
	}
    }


  if ( ! run_reliability (ds, &reliability)) 
    goto error;

  free (reliability.variables);
  return CMD_SUCCESS;

 error:
  free (reliability.variables);
  return CMD_FAILURE;
}
Example #14
0
Tokens jsonnet_lex(const std::string &filename, const char *input)
{
    unsigned long line_number = 1;
    const char *line_start = input;

    Tokens r;

    const char *c = input;

    Fodder fodder;
    bool fresh_line = true;  // Are we tokenizing from the beginning of a new line?

    while (*c!='\0') {
        Token::Kind kind;
        std::string data;
        std::string string_block_indent;
        std::string string_block_term_indent;

        unsigned new_lines, indent;
        lex_ws(c, new_lines, indent, line_start, line_number);

        // If it's the end of the file, discard final whitespace.
        if (*c == '\0')
            break;

        if (new_lines > 0) {
            // Otherwise store whitespace in fodder.
            unsigned blanks = new_lines - 1;
            fodder.emplace_back(FodderElement::LINE_END, blanks, indent, EMPTY);
            fresh_line = true;
        }

        Location begin(line_number, c - line_start + 1);

        switch (*c) {

            // The following operators should never be combined with subsequent symbols.
            case '{':
            kind = Token::BRACE_L;
            c++;
            break;

            case '}':
            kind = Token::BRACE_R;
            c++;
            break;

            case '[':
            kind = Token::BRACKET_L;
            c++;
            break;

            case ']':
            kind = Token::BRACKET_R;
            c++;
            break;

            case ',':
            kind = Token::COMMA;
            c++;
            break;

            case '.':
            kind = Token::DOT;
            c++;
            break;

            case '(':
            kind = Token::PAREN_L;
            c++;
            break;

            case ')':
            kind = Token::PAREN_R;
            c++;
            break;

            case ';':
            kind = Token::SEMICOLON;
            c++;
            break;

            // Numeric literals.
            case '0': case '1': case '2': case '3': case '4':
            case '5': case '6': case '7': case '8': case '9':
            kind = Token::NUMBER;
            data = lex_number(c, filename, begin);
            break;

            // String literals.
            case '"': {
                c++;
                for (; ; ++c) {
                    if (*c == '\0') {
                        throw StaticError(filename, begin, "Unterminated string");
                    }
                    if (*c == '"') {
                        break;
                    }
                    if (*c == '\\' && *(c+1) != '\0') {
                        data += *c;
                        ++c;
                    }
                    if (*c == '\n') {
                        // Maintain line/column counters.
                        line_number++;
                        line_start = c+1;
                    }
                    data += *c;
                }
                c++;  // Advance beyond the ".
                kind = Token::STRING_DOUBLE;
            }
            break;

            // String literals.
            case '\'': {
                c++;
                for (; ; ++c) {
                    if (*c == '\0') {
                        throw StaticError(filename, begin, "Unterminated string");
                    }
                    if (*c == '\'') {
                        break;
                    }
                    if (*c == '\\' && *(c+1) != '\0') {
                        data += *c;
                        ++c;
                    }
                    if (*c == '\n') {
                        // Maintain line/column counters.
                        line_number++;
                        line_start = c+1;
                    }
                    data += *c;
                }
                c++;  // Advance beyond the '.
                kind = Token::STRING_SINGLE;
            }
            break;

            // Keywords
            default:
            if (is_identifier_first(*c)) {
                std::string id;
                for (; is_identifier(*c); ++c)
                    id += *c;
                if (id == "assert") {
                    kind = Token::ASSERT;
                } else if (id == "else") {
                    kind = Token::ELSE;
                } else if (id == "error") {
                    kind = Token::ERROR;
                } else if (id == "false") {
                    kind = Token::FALSE;
                } else if (id == "for") {
                    kind = Token::FOR;
                } else if (id == "function") {
                    kind = Token::FUNCTION;
                } else if (id == "if") {
                    kind = Token::IF;
                } else if (id == "import") {
                    kind = Token::IMPORT;
                } else if (id == "importstr") {
                    kind = Token::IMPORTSTR;
                } else if (id == "in") {
                    kind = Token::IN;
                } else if (id == "local") {
                    kind = Token::LOCAL;
                } else if (id == "null") {
                    kind = Token::NULL_LIT;
                } else if (id == "self") {
                    kind = Token::SELF;
                } else if (id == "super") {
                    kind = Token::SUPER;
                } else if (id == "tailstrict") {
                    kind = Token::TAILSTRICT;
                } else if (id == "then") {
                    kind = Token::THEN;
                } else if (id == "true") {
                    kind = Token::TRUE;
                } else {
                    // Not a keyword, must be an identifier.
                    kind = Token::IDENTIFIER;
                }
                data = id;

            } else if (is_symbol(*c) || *c == '#') {

                // Single line C++ and Python style comments.
                if (*c == '#' || (*c == '/' && *(c+1) == '/')) {
                    std::vector<std::string> comment(1);
                    unsigned blanks;
                    unsigned indent;
                    lex_until_newline(c, comment[0], blanks, indent, line_start, line_number);
                    auto kind = fresh_line ? FodderElement::PARAGRAPH : FodderElement::LINE_END;
                    fodder.emplace_back(kind, blanks, indent, comment);
                    fresh_line = true;
                    continue;  // We've not got a token, just fodder, so keep scanning.
                }

                // Multi-line C style comment.
                if (*c == '/' && *(c+1) == '*') {

                    unsigned margin = c - line_start;
 
                    const char *initial_c = c;
                    c += 2;  // Avoid matching /*/: skip the /* before starting the search for */.

                    while (!(*c == '*' && *(c+1) == '/')) {
                        if (*c == '\0') {
                            auto msg = "Multi-line comment has no terminating */.";
                            throw StaticError(filename, begin, msg);
                        }
                        if (*c == '\n') {
                            // Just keep track of the line / column counters.
                            line_number++;
                            line_start = c+1;
                        }
                        ++c;
                    }
                    c += 2;  // Move the pointer to the char after the closing '/'.

                    std::string comment(initial_c, c - initial_c);  // Includes the "/*" and "*/".

                    // Lex whitespace after comment
                    unsigned new_lines_after, indent_after;
                    lex_ws(c, new_lines_after, indent_after, line_start, line_number);
                    std::vector<std::string> lines;
                    if (comment.find('\n') >= comment.length()) {
                        // Comment looks like /* foo */
                        lines.push_back(comment);
                        fodder.emplace_back(FodderElement::INTERSTITIAL, 0, 0, lines);
                        if (new_lines_after > 0) {
                            fodder.emplace_back(FodderElement::LINE_END, new_lines_after - 1,
                                                indent_after, EMPTY);
                            fresh_line = true;
                        }
                    } else {
                        lines = line_split(comment, margin);
                        assert(lines[0][0] == '/');
                        // Little hack to support PARAGRAPHs with * down the LHS:
                        // Add a space to lines that start with a '*'
                        bool all_star = true;
                        for (auto &l : lines) {
                            if (l[0] != '*')
                                all_star = false;
                        }
                        if (all_star) {
                            for (auto &l : lines) {
                                if (l[0] == '*') l = " " + l;
                            }
                        }
                        if (new_lines_after == 0) {
                            // Ensure a line end after the paragraph.
                            new_lines_after = 1;
                            indent_after = 0;
                        }
                        if (!fresh_line)
                            // Ensure a line end before the comment.
                            fodder.emplace_back(FodderElement::LINE_END, 0, 0, EMPTY);
                        fodder.emplace_back(FodderElement::PARAGRAPH, new_lines_after - 1,
                                            indent_after, lines);
                        fresh_line = true;
                    }
                    continue;  // We've not got a token, just fodder, so keep scanning.
                }

                // Text block
                if (*c == '|' && *(c+1) == '|' && *(c+2) == '|' && *(c+3) == '\n') {
                    std::stringstream block;
                    c += 4; // Skip the "|||\n"
                    line_number++;
                    // Skip any blank lines at the beginning of the block.
                    while (*c == '\n') {
                        line_number++;
                        ++c;
                        block << '\n';
                    }
                    line_start = c;
                    const char *first_line = c;
                    int ws_chars = whitespace_check(first_line, c);
                    string_block_indent = std::string(first_line, ws_chars);
                    if (ws_chars == 0) {
                        auto msg = "Text block's first line must start with whitespace.";
                        throw StaticError(filename, begin, msg);
                    }
                    while (true) {
                        assert(ws_chars > 0);
                        // Read up to the \n
                        for (c = &c[ws_chars]; *c != '\n' ; ++c) {
                            if (*c == '\0')
                                throw StaticError(filename, begin, "Unexpected EOF");
                            block << *c;
                        }
                        // Add the \n
                        block << '\n';
                        ++c;
                        line_number++;
                        line_start = c;
                        // Skip any blank lines
                        while (*c == '\n') {
                            line_number++;
                            ++c;
                            block << '\n';
                        }
                        // Examine next line
                        ws_chars = whitespace_check(first_line, c);
                        if (ws_chars == 0) {
                            // End of text block
                            // Skip over any whitespace
                            while (*c == ' ' || *c == '\t') {
                                string_block_term_indent += *c;
                                ++c;
                            }
                            // Expect |||
                            if (!(*c == '|' && *(c+1) == '|' && *(c+2) == '|')) {
                                auto msg = "Text block not terminated with |||";
                                throw StaticError(filename, begin, msg);
                            }
                            c += 3;  // Leave after the last |
                            data = block.str();
                            kind = Token::STRING_BLOCK;
                            break;  // Out of the while loop.
                        }
                    }

                    break;  // Out of the switch.
                }

                const char *operator_begin = c;
                for (; is_symbol(*c) ; ++c) {
                    // Not allowed // in operators
                    if (*c == '/' && *(c+1) == '/') break;
                    // Not allowed /* in operators
                    if (*c == '/' && *(c+1) == '*') break;
                    // Not allowed ||| in operators
                    if (*c == '|' && *(c+1) == '|' && *(c+2) == '|') break;
                }
                // Not allowed to end with a + - ~ ! unless a single char.
                // So, wind it back if we need to (but not too far).
                while (c > operator_begin + 1
                       && (*(c-1) == '+' || *(c-1) == '-' || *(c-1) == '~' || *(c-1) == '!')) {
                    c--;
                }
                data += std::string(operator_begin, c);
                if (data == "$") {
                    kind = Token::DOLLAR;
                    data = "";
                } else {
                    kind = Token::OPERATOR;
                }
            } else {
                std::stringstream ss;
                ss << "Could not lex the character ";
                auto uc = (unsigned char)(*c);
                if (*c < 32)
                    ss << "code " << unsigned(uc);
                else
                    ss << "'" << *c << "'";
                throw StaticError(filename, begin, ss.str());
            }
        }

        Location end(line_number, c - line_start);
        r.emplace_back(kind, fodder, data, string_block_indent, string_block_term_indent,
                       LocationRange(filename, begin, end));
        fodder.clear();
        fresh_line = false;
    }

    Location end(line_number, c - line_start + 1);
    r.emplace_back(Token::END_OF_FILE, fodder, "", "", "", LocationRange(filename, end, end));
    return r;
}