void align_struct_initializers(void)
{
   LOG_FUNC_ENTRY();
   chunk_t *pc = chunk_get_head();
   while (pc != nullptr)
   {
      chunk_t *prev = chunk_get_prev_ncnl(pc);
      if (  chunk_is_token(prev, CT_ASSIGN)
         && (  chunk_is_token(pc, CT_BRACE_OPEN)
            || (language_is_set(LANG_D) && chunk_is_token(pc, CT_SQUARE_OPEN))))
      {
         align_init_brace(pc);
      }
      pc = chunk_get_next_type(pc, CT_BRACE_OPEN, -1);
   }
} // align_struct_initializers
示例#2
0
void align_add(ChunkStack &cs, chunk_t *pc, size_t &max_col)
{
   LOG_FUNC_ENTRY();

   size_t  min_col;
   chunk_t *prev = chunk_get_prev(pc);
   if (prev == nullptr || chunk_is_newline(prev))
   {
      min_col = 1;
      LOG_FMT(LALADD, "%s(%d): pc->orig_line=%zu, pc->col=%zu max_col=%zu min_col=%zu\n",
              __func__, __LINE__, pc->orig_line, pc->column, max_col, min_col);
   }
   else
   {
      if (chunk_is_token(prev, CT_COMMENT_MULTI))
      {
         min_col = prev->orig_col_end + 1;
      }
      else
      {
         min_col = prev->column + prev->len() + 1;
      }
      LOG_FMT(LALADD, "%s(%d): pc->orig_line=%zu, pc->col=%zu max_col=%zu min_col=%zu multi:%s prev->col=%zu prev->len()=%zu %s\n",
              __func__, __LINE__, pc->orig_line, pc->column, max_col, min_col, (chunk_is_token(prev, CT_COMMENT_MULTI)) ? "Y" : "N",
              (chunk_is_token(prev, CT_COMMENT_MULTI)) ? prev->orig_col_end : (UINT32)prev->column, prev->len(), get_token_name(prev->type));
   }

   if (cs.Empty())
   {
      max_col = 0;
   }

   cs.Push_Back(pc);
   if (min_col > max_col)
   {
      max_col = min_col;
   }
} // align_add
示例#3
0
void tokenize_cleanup(void)
{
   LOG_FUNC_ENTRY();

   chunk_t *pc   = chunk_get_head();
   chunk_t *prev = NULL;
   chunk_t *next;
   chunk_t *tmp;
   chunk_t *tmp2;
   bool    in_type_cast = false;

   cpd.unc_stage = US_TOKENIZE_CLEANUP;

   /* Since [] is expected to be TSQUARE for the 'operator', we need to make
    * this change in the first pass.
    */
   for (pc = chunk_get_head(); pc != NULL; pc = chunk_get_next_ncnl(pc))
   {
      if (pc->type == CT_SQUARE_OPEN)
      {
         next = chunk_get_next_ncnl(pc);
         if (chunk_is_token(next, CT_SQUARE_CLOSE))
         {
            /* Change '[' + ']' into '[]' */
            set_chunk_type(pc, CT_TSQUARE);
            pc->str = "[]";
            // bug # 664
            // The original orig_col_end of CT_SQUARE_CLOSE is stored at orig_col_end of CT_TSQUARE.
            // pc->orig_col_end += 1;
            pc->orig_col_end = next->orig_col_end;
            chunk_del(next);
         }
      }
      if ((pc->type == CT_SEMICOLON) &&
          (pc->flags & PCF_IN_PREPROC) &&
          !chunk_get_next_ncnl(pc, CNAV_PREPROC))
      {
         LOG_FMT(LNOTE, "%s:%d Detected a macro that ends with a semicolon. Possible failures if used.\n",
                 cpd.filename, pc->orig_line);
      }
   }

   /* We can handle everything else in the second pass */
   pc   = chunk_get_head();
   next = chunk_get_next_ncnl(pc);
   while ((pc != NULL) && (next != NULL))
   {
      if ((pc->type == CT_DOT) && (cpd.lang_flags & LANG_ALLC))
      {
         set_chunk_type(pc, CT_MEMBER);
      }

      if ((pc->type == CT_NULLCOND) && (cpd.lang_flags & LANG_CS))
      {
         set_chunk_type(pc, CT_MEMBER);
      }

      /* Determine the version stuff (D only) */
      if (pc->type == CT_D_VERSION)
      {
         if (next->type == CT_PAREN_OPEN)
         {
            set_chunk_type(pc, CT_D_VERSION_IF);
         }
         else
         {
            if (next->type != CT_ASSIGN)
            {
               LOG_FMT(LERR, "%s:%d %s: version: Unexpected token %s\n",
                       cpd.filename, pc->orig_line, __func__, get_token_name(next->type));
               cpd.error_count++;
            }
            set_chunk_type(pc, CT_WORD);
         }
      }

      /* Determine the scope stuff (D only) */
      if (pc->type == CT_D_SCOPE)
      {
         if (next->type == CT_PAREN_OPEN)
         {
            set_chunk_type(pc, CT_D_SCOPE_IF);
         }
         else
         {
            set_chunk_type(pc, CT_TYPE);
         }
      }

      /**
       * Change CT_BASE before CT_PAREN_OPEN to CT_WORD.
       * public myclass() : base() {
       * }
       */
      if ((pc->type == CT_BASE) && (next->type == CT_PAREN_OPEN))
      {
         set_chunk_type(pc, CT_WORD);
      }

      if ((pc->type == CT_ENUM) && (next->type == CT_CLASS))
      {
         set_chunk_type(next, CT_ENUM_CLASS);
      }

      /**
       * Change CT_WORD after CT_ENUM, CT_UNION, or CT_STRUCT to CT_TYPE
       * Change CT_WORD before CT_WORD to CT_TYPE
       */
      if (next->type == CT_WORD)
      {
         if ((pc->type == CT_ENUM) ||
             (pc->type == CT_ENUM_CLASS) ||
             (pc->type == CT_UNION) ||
             (pc->type == CT_STRUCT))
         {
            set_chunk_type(next, CT_TYPE);
         }
         if (pc->type == CT_WORD)
         {
            set_chunk_type(pc, CT_TYPE);
         }
      }

      /* change extern to qualifier if extern isn't followed by a string or
       * an open paren
       */
      if (pc->type == CT_EXTERN)
      {
         if (next->type == CT_STRING)
         {
            /* Probably 'extern "C"' */
         }
         else if (next->type == CT_PAREN_OPEN)
         {
            /* Probably 'extern (C)' */
         }
         else
         {
            /* Something else followed by a open brace */
            tmp = chunk_get_next_ncnl(next);
            if ((tmp == NULL) || (tmp->type != CT_BRACE_OPEN))
            {
               set_chunk_type(pc, CT_QUALIFIER);
            }
         }
      }

      /**
       * Change CT_STAR to CT_PTR_TYPE if preceded by CT_TYPE,
       * CT_QUALIFIER, or CT_PTR_TYPE.
       */
      if ((next->type == CT_STAR) &&
          ((pc->type == CT_TYPE) ||
           (pc->type == CT_QUALIFIER) ||
           (pc->type == CT_PTR_TYPE)))
      {
         set_chunk_type(next, CT_PTR_TYPE);
      }

      if ((pc->type == CT_TYPE_CAST) &&
          (next->type == CT_ANGLE_OPEN))
      {
         set_chunk_parent(next, CT_TYPE_CAST);
         in_type_cast = true;
      }

      /**
       * Change angle open/close to CT_COMPARE, if not a template thingy
       */
      if ((pc->type == CT_ANGLE_OPEN) && (pc->parent_type != CT_TYPE_CAST))
      {
         /* pretty much all languages except C use <> for something other than
          * comparisons.  "#include<xxx>" is handled elsewhere.
          */
         if (cpd.lang_flags & (LANG_CPP | LANG_CS | LANG_JAVA | LANG_VALA | LANG_OC))
         {
            // bug #663
            check_template(pc);
         }
         else
         {
            /* convert CT_ANGLE_OPEN to CT_COMPARE */
            set_chunk_type(pc, CT_COMPARE);
         }
      }
      if ((pc->type == CT_ANGLE_CLOSE) && (pc->parent_type != CT_TEMPLATE))
      {
         if (in_type_cast)
         {
            in_type_cast = false;
            set_chunk_parent(pc, CT_TYPE_CAST);
         }
         else
         {
            next = handle_double_angle_close(pc);
         }
      }

      if (cpd.lang_flags & LANG_D)
      {
         /* Check for the D string concat symbol '~' */
         if ((pc->type == CT_INV) &&
             ((prev->type == CT_STRING) ||
              (prev->type == CT_WORD) ||
              (next->type == CT_STRING)))
         {
            set_chunk_type(pc, CT_CONCAT);
         }

         /* Check for the D template symbol '!' (word + '!' + word or '(') */
         if ((pc->type == CT_NOT) &&
             (prev->type == CT_WORD) &&
             ((next->type == CT_PAREN_OPEN) ||
              (next->type == CT_WORD) ||
              (next->type == CT_TYPE)))
         {
            set_chunk_type(pc, CT_D_TEMPLATE);
         }

         /* handle "version(unittest) { }" vs "unittest { }" */
         if (prev && (pc->type == CT_UNITTEST) && (prev->type == CT_PAREN_OPEN))
         {
            set_chunk_type(pc, CT_WORD);
         }

         /* handle 'static if' and merge the tokens */
         if (prev && (pc->type == CT_IF) && chunk_is_str(prev, "static", 6))
         {
            /* delete PREV and merge with IF */
            pc->str.insert(0, ' ');
            pc->str.insert(0, prev->str);
            pc->orig_col  = prev->orig_col;
            pc->orig_line = prev->orig_line;
            chunk_t *to_be_deleted = prev;
            prev = chunk_get_prev_ncnl(prev);
            chunk_del(to_be_deleted);
         }
      }

      if (cpd.lang_flags & LANG_CPP)
      {
         /* Change Word before '::' into a type */
         if ((pc->type == CT_WORD) && (next->type == CT_DC_MEMBER))
         {
            set_chunk_type(pc, CT_TYPE);
         }
      }

      /* Change get/set to CT_WORD if not followed by a brace open */
      if ((pc->type == CT_GETSET) && (next->type != CT_BRACE_OPEN))
      {
         if ((next->type == CT_SEMICOLON) &&
             ((prev->type == CT_BRACE_CLOSE) ||
              (prev->type == CT_BRACE_OPEN) ||
              (prev->type == CT_SEMICOLON)))
         {
            set_chunk_type(pc, CT_GETSET_EMPTY);
            set_chunk_parent(next, CT_GETSET);
         }
         else
         {
            set_chunk_type(pc, CT_WORD);
         }
      }

      /* Interface is only a keyword in MS land if followed by 'class' or 'struct'
       * likewise, 'class' may be a member name in Java.
       */
      if ((pc->type == CT_CLASS) && !CharTable::IsKw1(next->str[0]))
      {
         set_chunk_type(pc, CT_WORD);
      }

      /* Change item after operator (>=, ==, etc) to a CT_OPERATOR_VAL
       * Usually the next item is part of the operator.
       * In a few cases the next few tokens are part of it:
       *  operator +       - common case
       *  operator >>      - need to combine '>' and '>'
       *  operator ()
       *  operator []      - already converted to TSQUARE
       *  operator new []
       *  operator delete []
       *  operator const char *
       *  operator const B&
       *  operator std::allocator<U>
       *
       * In all cases except the last, this will put the entire operator value
       * in one chunk.
       */
      if (pc->type == CT_OPERATOR)
      {
         tmp2 = chunk_get_next(next);
         /* Handle special case of () operator -- [] already handled */
         if (next->type == CT_PAREN_OPEN)
         {
            tmp = chunk_get_next(next);
            if ((tmp != NULL) && (tmp->type == CT_PAREN_CLOSE))
            {
               next->str = "()";
               set_chunk_type(next, CT_OPERATOR_VAL);
               chunk_del(tmp);
               next->orig_col_end += 1;
            }
         }
         else if ((next->type == CT_ANGLE_CLOSE) &&
                  tmp2 && (tmp2->type == CT_ANGLE_CLOSE) &&
                  (tmp2->orig_col == next->orig_col_end))
         {
            next->str.append('>');
            next->orig_col_end++;
            set_chunk_type(next, CT_OPERATOR_VAL);
            chunk_del(tmp2);
         }
         else if (next->flags & PCF_PUNCTUATOR)
         {
            set_chunk_type(next, CT_OPERATOR_VAL);
         }
         else
         {
            set_chunk_type(next, CT_TYPE);

            /* Replace next with a collection of all tokens that are part of
             * the type.
             */
            tmp2 = next;
            while ((tmp = chunk_get_next(tmp2)) != NULL)
            {
               if ((tmp->type != CT_WORD) &&
                   (tmp->type != CT_TYPE) &&
                   (tmp->type != CT_QUALIFIER) &&
                   (tmp->type != CT_STAR) &&
                   (tmp->type != CT_CARET) &&
                   (tmp->type != CT_AMP) &&
                   (tmp->type != CT_TSQUARE))
               {
                  break;
               }
               /* Change tmp into a type so that space_needed() works right */
               make_type(tmp);
               int num_sp = space_needed(tmp2, tmp);
               while (num_sp-- > 0)
               {
                  next->str.append(" ");
               }
               next->str.append(tmp->str);
               tmp2 = tmp;
            }

            while ((tmp2 = chunk_get_next(next)) != tmp)
            {
               chunk_del(tmp2);
            }

            set_chunk_type(next, CT_OPERATOR_VAL);

            next->orig_col_end = next->orig_col + next->len();
         }
         set_chunk_parent(next, CT_OPERATOR);

         LOG_FMT(LOPERATOR, "%s: %d:%d operator '%s'\n",
                 __func__, pc->orig_line, pc->orig_col, next->text());
      }

      /* Change private, public, protected into either a qualifier or label */
      if (pc->type == CT_PRIVATE)
      {
         /* Handle Qt slots - maybe should just check for a CT_WORD? */
         if (chunk_is_str(next, "slots", 5) || chunk_is_str(next, "Q_SLOTS", 7))
         {
            tmp = chunk_get_next(next);
            if ((tmp != NULL) && (tmp->type == CT_COLON))
            {
               next = tmp;
            }
         }
         if (next->type == CT_COLON)
         {
            set_chunk_type(next, CT_PRIVATE_COLON);
            if ((tmp = chunk_get_next_ncnl(next)) != NULL)
            {
               chunk_flags_set(tmp, PCF_STMT_START | PCF_EXPR_START);
            }
         }
         else
         {
            set_chunk_type(pc, (chunk_is_str(pc, "signals", 7) || chunk_is_str(pc, "Q_SIGNALS", 9)) ? CT_WORD : CT_QUALIFIER);
         }
      }

      /* Look for <newline> 'EXEC' 'SQL' */
      if ((chunk_is_str(pc, "EXEC", 4) && chunk_is_str(next, "SQL", 3)) ||
          ((*pc->str == '$') && (pc->type != CT_SQL_WORD)))
      {
         tmp = chunk_get_prev(pc);
         if (chunk_is_newline(tmp))
         {
            if (*pc->str == '$')
            {
               set_chunk_type(pc, CT_SQL_EXEC);
               if (pc->len() > 1)
               {
                  /* SPLIT OFF '$' */
                  chunk_t nc;

                  nc = *pc;
                  pc->str.resize(1);
                  pc->orig_col_end = pc->orig_col + 1;

                  nc.type = CT_SQL_WORD;
                  nc.str.pop_front();
                  nc.orig_col++;
                  nc.column++;
                  chunk_add_after(&nc, pc);

                  next = chunk_get_next(pc);
               }
            }
            tmp = chunk_get_next(next);
            if (chunk_is_str_case(tmp, "BEGIN", 5))
            {
               set_chunk_type(pc, CT_SQL_BEGIN);
            }
            else if (chunk_is_str_case(tmp, "END", 3))
            {
               set_chunk_type(pc, CT_SQL_END);
            }
            else
            {
               set_chunk_type(pc, CT_SQL_EXEC);
            }

            /* Change words into CT_SQL_WORD until CT_SEMICOLON */
            while (tmp != NULL)
            {
               if (tmp->type == CT_SEMICOLON)
               {
                  break;
               }
               if ((tmp->len() > 0) && (unc_isalpha(*tmp->str) || (*tmp->str == '$')))
               {
                  set_chunk_type(tmp, CT_SQL_WORD);
               }
               tmp = chunk_get_next_ncnl(tmp);
            }
         }
      }

      /* handle MS abomination 'for each' */
      if ((pc->type == CT_FOR) && chunk_is_str(next, "each", 4) &&
          (next == chunk_get_next(pc)))
      {
         /* merge the two with a space between */
         pc->str.append(' ');
         pc->str         += next->str;
         pc->orig_col_end = next->orig_col_end;
         chunk_del(next);
         next = chunk_get_next_ncnl(pc);
         /* label the 'in' */
         if (next && (next->type == CT_PAREN_OPEN))
         {
            tmp = chunk_get_next_ncnl(next);
            while (tmp && (tmp->type != CT_PAREN_CLOSE))
            {
               if (chunk_is_str(tmp, "in", 2))
               {
                  set_chunk_type(tmp, CT_IN);
                  break;
               }
               tmp = chunk_get_next_ncnl(tmp);
            }
         }
      }

      /* ObjectiveC allows keywords to be used as identifiers in some situations
       * This is a dirty hack to allow some of the more common situations.
       */
      if (cpd.lang_flags & LANG_OC)
      {
         if (((pc->type == CT_IF) ||
              (pc->type == CT_FOR) ||
              (pc->type == CT_WHILE)) &&
             !chunk_is_token(next, CT_PAREN_OPEN))
         {
            set_chunk_type(pc, CT_WORD);
         }
         if ((pc->type == CT_DO) &&
             (chunk_is_token(prev, CT_MINUS) ||
              chunk_is_token(next, CT_SQUARE_CLOSE)))
         {
            set_chunk_type(pc, CT_WORD);
         }
      }

      /* Another hack to clean up more keyword abuse */
      if ((pc->type == CT_CLASS) &&
          (chunk_is_token(prev, CT_DOT) ||
           chunk_is_token(next, CT_DOT)))
      {
         set_chunk_type(pc, CT_WORD);
      }

      /* Detect Objective C class name */
      if ((pc->type == CT_OC_IMPL) ||
          (pc->type == CT_OC_INTF) ||
          (pc->type == CT_OC_PROTOCOL))
      {
         if (next->type != CT_PAREN_OPEN)
         {
            set_chunk_type(next, CT_OC_CLASS);
         }
         set_chunk_parent(next, pc->type);

         tmp = chunk_get_next_ncnl(next);
         if (tmp != NULL)
         {
            chunk_flags_set(tmp, PCF_STMT_START | PCF_EXPR_START);
         }

         tmp = chunk_get_next_type(pc, CT_OC_END, pc->level);
         if (tmp != NULL)
         {
            set_chunk_parent(tmp, pc->type);
         }
      }

      if (pc->type == CT_OC_INTF)
      {
         tmp = chunk_get_next_ncnl(pc, CNAV_PREPROC);
         while ((tmp != NULL) && (tmp->type != CT_OC_END))
         {
            if (get_token_pattern_class(tmp->type) != PATCLS_NONE)
            {
               LOG_FMT(LOBJCWORD, "@interface %d:%d change '%s' (%s) to CT_WORD\n",
                       pc->orig_line, pc->orig_col, tmp->text(),
                       get_token_name(tmp->type));
               set_chunk_type(tmp, CT_WORD);
            }
            tmp = chunk_get_next_ncnl(tmp, CNAV_PREPROC);
         }
      }

      /* Detect Objective-C categories and class extensions */
      /* @interface ClassName (CategoryName) */
      /* @implementation ClassName (CategoryName) */
      /* @interface ClassName () */
      /* @implementation ClassName () */
      if (((pc->parent_type == CT_OC_IMPL) ||
           (pc->parent_type == CT_OC_INTF) ||
           (pc->type == CT_OC_CLASS)) &&
          (next->type == CT_PAREN_OPEN))
      {
         set_chunk_parent(next, pc->parent_type);

         tmp = chunk_get_next(next);
         if ((tmp != NULL) && (tmp->next != NULL))
         {
            if (tmp->type == CT_PAREN_CLOSE)
            {
               //set_chunk_type(tmp, CT_OC_CLASS_EXT);
               set_chunk_parent(tmp, pc->parent_type);
            }
            else
            {
               set_chunk_type(tmp, CT_OC_CATEGORY);
               set_chunk_parent(tmp, pc->parent_type);
            }
         }

         tmp = chunk_get_next_type(pc, CT_PAREN_CLOSE, pc->level);
         if (tmp != NULL)
         {
            set_chunk_parent(tmp, pc->parent_type);
         }
      }

      /* Detect Objective C @property
       *  @property NSString *stringProperty;
       *  @property(nonatomic, retain) NSMutableDictionary *shareWith;
       */
      if (pc->type == CT_OC_PROPERTY)
      {
         if (next->type != CT_PAREN_OPEN)
         {
            chunk_flags_set(next, PCF_STMT_START | PCF_EXPR_START);
         }
         else
         {
            set_chunk_parent(next, pc->type);

            tmp = chunk_get_next_type(pc, CT_PAREN_CLOSE, pc->level);
            if (tmp != NULL)
            {
               set_chunk_parent(tmp, pc->type);
               tmp = chunk_get_next_ncnl(tmp);
               if (tmp != NULL)
               {
                  chunk_flags_set(tmp, PCF_STMT_START | PCF_EXPR_START);

                  tmp = chunk_get_next_type(tmp, CT_SEMICOLON, pc->level);
                  if (tmp != NULL)
                  {
                     set_chunk_parent(tmp, pc->type);
                  }
               }
            }
         }
      }

      /* Detect Objective C @selector
       *  @selector(msgNameWithNoArg)
       *  @selector(msgNameWith1Arg:)
       *  @selector(msgNameWith2Args:arg2Name:)
       */
      if ((pc->type == CT_OC_SEL) && (next->type == CT_PAREN_OPEN))
      {
         set_chunk_parent(next, pc->type);

         tmp = chunk_get_next(next);
         if (tmp != NULL)
         {
            set_chunk_type(tmp, CT_OC_SEL_NAME);
            set_chunk_parent(tmp, pc->type);

            while ((tmp = chunk_get_next_ncnl(tmp)) != NULL)
            {
               if (tmp->type == CT_PAREN_CLOSE)
               {
                  set_chunk_parent(tmp, CT_OC_SEL);
                  break;
               }
               set_chunk_type(tmp, CT_OC_SEL_NAME);
               set_chunk_parent(tmp, pc->type);
            }
         }
      }

      /* Handle special preprocessor junk */
      if (pc->type == CT_PREPROC)
      {
         set_chunk_parent(pc, next->type);
      }

      /* Detect "pragma region" and "pragma endregion" */
      if ((pc->type == CT_PP_PRAGMA) && (next->type == CT_PREPROC_BODY))
      {
         if ((memcmp(next->str, "region", 6) == 0) ||
             (memcmp(next->str, "endregion", 9) == 0))
         {
            set_chunk_type(pc, (*next->str == 'r') ? CT_PP_REGION : CT_PP_ENDREGION);

            set_chunk_parent(prev, pc->type);
         }
      }

      /* Check for C# nullable types '?' is in next */
      if ((cpd.lang_flags & LANG_CS) &&
          (next->type == CT_QUESTION) &&
          (next->orig_col == (pc->orig_col + pc->len())))
      {
         tmp = chunk_get_next_ncnl(next);
         if (tmp != NULL)
         {
            bool doit = ((tmp->type == CT_PAREN_CLOSE) ||
                         (tmp->type == CT_ANGLE_CLOSE));

            if (tmp->type == CT_WORD)
            {
               tmp2 = chunk_get_next_ncnl(tmp);
               if ((tmp2 != NULL) &&
                   ((tmp2->type == CT_SEMICOLON) ||
                    (tmp2->type == CT_ASSIGN) ||
                    (tmp2->type == CT_COMMA) ||
                    (tmp2->type == CT_BRACE_OPEN)))
               {
                  doit = true;
               }
            }

            if (doit)
            {
               pc->str         += next->str;
               pc->orig_col_end = next->orig_col_end;
               chunk_del(next);
               next = tmp;
            }
         }
      }

      /* Change 'default(' into a sizeof-like statement */
      if ((cpd.lang_flags & LANG_CS) &&
          (pc->type == CT_DEFAULT) &&
          (next->type == CT_PAREN_OPEN))
      {
         set_chunk_type(pc, CT_SIZEOF);
      }

      if ((pc->type == CT_UNSAFE) && (next->type != CT_BRACE_OPEN))
      {
         set_chunk_type(pc, CT_QUALIFIER);
      }

      if (((pc->type == CT_USING) ||
           ((pc->type == CT_TRY) && (cpd.lang_flags & LANG_JAVA))) &&
          (next->type == CT_PAREN_OPEN))
      {
         set_chunk_type(pc, CT_USING_STMT);
      }

      /* Add minimal support for C++0x rvalue references */
      if ((pc->type == CT_BOOL) && chunk_is_str(pc, "&&", 2))
      {
         if (prev->type == CT_TYPE)
         {
            set_chunk_type(pc, CT_BYREF);
         }
      }

      /* HACK: treat try followed by a colon as a qualifier to handle this:
       *   A::A(int) try : B() { } catch (...) { }
       */
      if ((pc->type == CT_TRY) && chunk_is_str(pc, "try", 3) &&
          (next != NULL) && (next->type == CT_COLON))
      {
         set_chunk_type(pc, CT_QUALIFIER);
      }

      /* If Java's 'synchronized' is in a method declaration, it should be
       * a qualifier. */
      if ((cpd.lang_flags & LANG_JAVA) &&
          (pc->type == CT_SYNCHRONIZED) &&
          (next->type != CT_PAREN_OPEN))
      {
         set_chunk_type(pc, CT_QUALIFIER);
      }

      // guy 2015-11-05
      // change CT_DC_MEMBER + CT_FOR into CT_DC_MEMBER + CT_FUNC_CALL
      if ((pc->type == CT_FOR) &&
          (pc->prev->type == CT_DC_MEMBER))
      {
         set_chunk_type(pc, CT_FUNC_CALL);
      }
      /* TODO: determine other stuff here */

      prev = pc;
      pc   = next;
      next = chunk_get_next_ncnl(pc);
   }
} // tokenize_cleanup
示例#4
0
文件: width.cpp 项目: Limsik/e17
/**
 * Scan backwards to find the most appropriate spot to split the line
 * and insert a newline.
 *
 * See if this needs special function handling.
 * Scan backwards and find the best token for the split.
 *
 * @param start The first chunk that exceeded the limit
 */
static void split_line(chunk_t *start)
{
   LOG_FMT(LSPLIT, "%s: line %d, col %d token:%s[%s] (IN_FUNC=%d) ",
           __func__, start->orig_line, start->column, start->str.c_str(),
           get_token_name(start->type),
           (start->flags & (PCF_IN_FCN_DEF | PCF_IN_FCN_CALL)) != 0);

   /**
    * break at maximum line length if indent_continue is absolute
    */
   if (cpd.settings[UO_indent_continue].n < 0)
   {
   }
   /* Check to see if we are in a for statement */
   else if ((start->flags & PCF_IN_FOR) != 0)
   {
      LOG_FMT(LSPLIT, " ** FOR SPLIT **\n");
      split_for_stmt(start);
      if (!is_past_width(start))
      {
         return;
      }
      LOG_FMT(LSPLIT, "%s: for split didn't work\n", __func__);
   }

   /* If this is in a function call or prototype, split on commas or right
    * after the open paren
    */
   else if (((start->flags & PCF_IN_FCN_DEF) != 0) ||
            ((start->level == (start->brace_level + 1)) &&
             ((start->flags & PCF_IN_FCN_CALL) != 0)))
   {
      LOG_FMT(LSPLIT, " ** FUNC SPLIT **\n");

      if (cpd.settings[UO_ls_func_split_full].b)
      {
         split_fcn_params_full(start);
         if (!is_past_width(start))
         {
            return;
         }
      }
      split_fcn_params(start);
      if (!is_past_width(start))
      {
         return;
      }
      LOG_FMT(LSPLIT, "%s: func split didn't work\n", __func__);
   }

   /**
    * Try to find the best spot to split the line
    */
   cw_entry ent;

   memset(&ent, 0, sizeof(ent));
   chunk_t *pc = start;
   chunk_t *prev;

   while (((pc = chunk_get_prev(pc)) != NULL) && !chunk_is_newline(pc))
   {
      if (pc->type != CT_SPACE)
      {
         try_split_here(ent, pc);
         // break at maximum line length if indent_continue is absolute
         if ((ent.pc != NULL) && (cpd.settings[UO_indent_continue].n < 0))
             break;
      }
   }

   if (ent.pc == NULL)
   {
      LOG_FMT(LSPLIT, "%s: TRY_SPLIT yielded NO SOLUTION for line %d at %s [%s]\n",
              __func__, start->orig_line, start->str.c_str(), get_token_name(start->type));
   }
   else
   {
      LOG_FMT(LSPLIT, "%s: TRY_SPLIT yielded '%s' [%s] on line %d\n", __func__,
              ent.pc->str.c_str(), get_token_name(ent.pc->type), ent.pc->orig_line);
   }

   /* Break before the token instead of after it according to the pos_xxx rules */
   if ((chunk_is_token(ent.pc, CT_ARITH) &&
        (cpd.settings[UO_pos_arith].tp & TP_LEAD)) ||
       (chunk_is_token(ent.pc, CT_ASSIGN) &&
        (cpd.settings[UO_pos_assign].tp & TP_LEAD)) ||
       (chunk_is_token(ent.pc, CT_COMPARE) &&
        (cpd.settings[UO_pos_compare].tp & TP_LEAD)) ||
       ((chunk_is_token(ent.pc, CT_COND_COLON) ||
         chunk_is_token(ent.pc, CT_QUESTION)) &&
        (cpd.settings[UO_pos_conditional].tp & TP_LEAD)) ||
       (chunk_is_token(ent.pc, CT_BOOL) &&
        (cpd.settings[UO_pos_bool].tp & TP_LEAD)))
   {
      pc = ent.pc;
   }
   else
   {
      pc = chunk_get_next(ent.pc);
   }
   if (pc == NULL)
   {
      pc = start;
      /* Don't break before a close, comma, or colon */
      if ((start->type == CT_PAREN_CLOSE) ||
          (start->type == CT_PAREN_OPEN) ||
          (start->type == CT_FPAREN_CLOSE) ||
          (start->type == CT_FPAREN_OPEN) ||
          (start->type == CT_SPAREN_CLOSE) ||
          (start->type == CT_SPAREN_OPEN) ||
          (start->type == CT_ANGLE_CLOSE) ||
          (start->type == CT_BRACE_CLOSE) ||
          (start->type == CT_COMMA) ||
          (start->type == CT_SEMICOLON) ||
          (start->type == CT_VSEMICOLON) ||
          (start->len() == 0))
      {
         LOG_FMT(LSPLIT, " ** NO GO **\n");

         /*TODO: Add in logic to handle 'hard' limits by backing up a token */
         return;
      }
   }

   /* add a newline before pc */
   prev = chunk_get_prev(pc);
   if ((prev != NULL) && !chunk_is_newline(pc) && !chunk_is_newline(prev))
   {
      int plen = (pc->len() < 5) ? pc->len() : 5;
      int slen = (start->len() < 5) ? start->len() : 5;
      LOG_FMT(LSPLIT, " '%.*s' [%s], started on token '%.*s' [%s]\n",
              plen, pc->str.c_str(), get_token_name(pc->type),
              slen, start->str.c_str(), get_token_name(start->type));

      split_before_chunk(pc);
   }
}
示例#5
0
static bool split_line(chunk_t *start)
{
   LOG_FUNC_ENTRY();
   LOG_FMT(LSPLIT, "%s(%d): start->text() '%s', orig_line is %zu, orig_col is %zu, type is %s\n",
           __func__, __LINE__, start->text(), start->orig_line, start->orig_col, get_token_name(start->type));
   LOG_FMT(LSPLIT, "   start->flags ");
   log_pcf_flags(LSPLIT, start->flags);
   LOG_FMT(LSPLIT, "   start->parent_type %s, (PCF_IN_FCN_DEF is %s), (PCF_IN_FCN_CALL is %s)\n",
           get_token_name(start->parent_type),
           ((start->flags & (PCF_IN_FCN_DEF)) != 0) ? "TRUE" : "FALSE",
           ((start->flags & (PCF_IN_FCN_CALL)) != 0) ? "TRUE" : "FALSE");

   // break at maximum line length if ls_code_width is true
   if (start->flags & PCF_ONE_LINER)
   {
      LOG_FMT(LSPLIT, "%s(%d): ** ONCE LINER SPLIT **\n", __func__, __LINE__);
      undo_one_liner(start);
      newlines_cleanup_braces(false);
      // Issue #1352
      cpd.changes++;
      return(false);
   }

   LOG_FMT(LSPLIT, "%s(%d):\n", __func__, __LINE__);
   if (cpd.settings[UO_ls_code_width].b)
   {
   }
   // Check to see if we are in a for statement
   else if (start->flags & PCF_IN_FOR)
   {
      LOG_FMT(LSPLIT, " ** FOR SPLIT **\n");
      split_for_stmt(start);
      if (!is_past_width(start))
      {
         return(true);
      }
      LOG_FMT(LSPLIT, "%s(%d): for split didn't work\n", __func__, __LINE__);
   }

   /*
    * If this is in a function call or prototype, split on commas or right
    * after the open parenthesis
    */
   else if (  (start->flags & PCF_IN_FCN_DEF)
           || start->parent_type == CT_FUNC_PROTO            // Issue #1169
           || (  (start->level == (start->brace_level + 1))
              && (start->flags & PCF_IN_FCN_CALL)))
   {
      LOG_FMT(LSPLIT, " ** FUNC SPLIT **\n");

      if (cpd.settings[UO_ls_func_split_full].b)
      {
         split_fcn_params_full(start);
         if (!is_past_width(start))
         {
            return(true);
         }
      }
      split_fcn_params(start);
      return(true);
   }

   /*
    * If this is in a template, split on commas, Issue #1170
    */
   else if (start->flags & PCF_IN_TEMPLATE)
   {
      LOG_FMT(LSPLIT, " ** TEMPLATE SPLIT **\n");
      split_template(start);
      return(true);
   }

   LOG_FMT(LSPLIT, "%s(%d):\n", __func__, __LINE__);
   // Try to find the best spot to split the line
   cw_entry ent;

   memset(&ent, 0, sizeof(ent));
   chunk_t *pc = start;
   chunk_t *prev;

   while (((pc = chunk_get_prev(pc)) != nullptr) && !chunk_is_newline(pc))
   {
      LOG_FMT(LSPLIT, "%s(%d): at %s, orig_line=%zu, orig_col=%zu\n",
              __func__, __LINE__, pc->text(), pc->orig_line, pc->orig_col);
      if (pc->type != CT_SPACE)
      {
         try_split_here(ent, pc);
         // break at maximum line length
         if (ent.pc != nullptr && (cpd.settings[UO_ls_code_width].b))
         {
            break;
         }
      }
   }

   if (ent.pc == nullptr)
   {
      LOG_FMT(LSPLIT, "\n%s(%d):    TRY_SPLIT yielded NO SOLUTION for orig_line %zu at '%s' [%s]\n",
              __func__, __LINE__, start->orig_line, start->text(), get_token_name(start->type));
   }
   else
   {
      LOG_FMT(LSPLIT, "\n%s(%d):    TRY_SPLIT yielded '%s' [%s] on orig_line %zu\n",
              __func__, __LINE__, ent.pc->text(), get_token_name(ent.pc->type), ent.pc->orig_line);
      LOG_FMT(LSPLIT, "%s(%d): ent at '%s', orig_col is %zu\n",
              __func__, __LINE__, ent.pc->text(), ent.pc->orig_col);
   }

   // Break before the token instead of after it according to the pos_xxx rules
   if (ent.pc == nullptr)
   {
      pc = nullptr;
   }
   else
   {
      if (  (  (  chunk_is_token(ent.pc, CT_ARITH)
               || chunk_is_token(ent.pc, CT_CARET))
            && (cpd.settings[UO_pos_arith].tp & TP_LEAD))
         || (  chunk_is_token(ent.pc, CT_ASSIGN)
            && (cpd.settings[UO_pos_assign].tp & TP_LEAD))
         || (  chunk_is_token(ent.pc, CT_COMPARE)
            && (cpd.settings[UO_pos_compare].tp & TP_LEAD))
         || (  (  chunk_is_token(ent.pc, CT_COND_COLON)
               || chunk_is_token(ent.pc, CT_QUESTION))
            && (cpd.settings[UO_pos_conditional].tp & TP_LEAD))
         || (  chunk_is_token(ent.pc, CT_BOOL)
            && (cpd.settings[UO_pos_bool].tp & TP_LEAD)))
      {
         pc = ent.pc;
      }
      else
      {
         pc = chunk_get_next(ent.pc);
      }
      LOG_FMT(LSPLIT, "%s(%d): at '%s', orig_col is %zu\n",
              __func__, __LINE__, pc->text(), pc->orig_col);
   }

   if (pc == nullptr)
   {
      pc = start;
      // Don't break before a close, comma, or colon
      if (  start->type == CT_PAREN_CLOSE
         || start->type == CT_PAREN_OPEN
         || start->type == CT_FPAREN_CLOSE
         || start->type == CT_FPAREN_OPEN
         || start->type == CT_SPAREN_CLOSE
         || start->type == CT_SPAREN_OPEN
         || start->type == CT_ANGLE_CLOSE
         || start->type == CT_BRACE_CLOSE
         || start->type == CT_COMMA
         || start->type == CT_SEMICOLON
         || start->type == CT_VSEMICOLON
         || start->len() == 0)
      {
         LOG_FMT(LSPLIT, " ** NO GO **\n");

         // TODO: Add in logic to handle 'hard' limits by backing up a token
         return(true);
      }
   }

   // add a newline before pc
   prev = chunk_get_prev(pc);
   if (  prev != nullptr
      && !chunk_is_newline(pc)
      && !chunk_is_newline(prev))
   {
      //int plen = (pc->len() < 5) ? pc->len() : 5;
      //int slen = (start->len() < 5) ? start->len() : 5;
      //LOG_FMT(LSPLIT, " '%.*s' [%s], started on token '%.*s' [%s]\n",
      //        plen, pc->text(), get_token_name(pc->type),
      //        slen, start->text(), get_token_name(start->type));
      LOG_FMT(LSPLIT, "%s(%d): text() '%s', type [%s], started on token '%s', type [%s]\n",
              __func__, __LINE__, pc->text(), get_token_name(pc->type),
              start->text(), get_token_name(start->type));

      split_before_chunk(pc);
   }
   return(true);
} // split_line
示例#6
0
/**
 * Adds an entry to the appropriate stack.
 *
 * @param pc      The chunk
 * @param seqnum  Optional seqnum (0=assign one)
 */
void AlignStack::Add(chunk_t *start, int seqnum)
{
   /* Assign a seqnum if needed */
   if (seqnum == 0)
   {
      seqnum = m_seqnum;
   }

   chunk_t *ali;
   chunk_t *ref;
   chunk_t *tmp;
   chunk_t *prev;
   chunk_t *next;

   int col_adj = 0;  /* Amount the column is shifted for 'dangle' mode */
   int tmp_col;
   int endcol;
   int gap;

   m_last_added = 0;

   /* Check threshold limits */
   if ((m_max_col == 0) || (m_thresh == 0) ||
       (((start->column + m_gap) <= (m_max_col + m_thresh)) &&
        (((start->column + m_gap) >= (m_max_col - m_thresh)) ||
         (start->column >= m_min_col))))
   {
      /* we are adding it, so update the newline seqnum */
      if (seqnum > m_nl_seqnum)
      {
         m_nl_seqnum = seqnum;
      }

      /**
       * SS_IGNORE: no special handling of '*' or '&', only 'foo' is aligned
       *     void     foo;  // gap=5, 'foo' is aligned
       *     char *   foo;  // gap=3, 'foo' is aligned
       *     foomatic foo;  // gap=1, 'foo' is aligned
       *  The gap is the columns between 'foo' and the previous token.
       *  [void - foo], ['*' - foo], etc
       *
       * SS_INCLUDE: - space between variable and '*' or '&' is eaten
       *     void     foo;  // gap=5, 'foo' is aligned
       *     char     *foo; // gap=5, '*' is aligned
       *     foomatic foo;  // gap=1, 'foo' is aligned
       *  The gap is the columns between the first '*' or '&' before foo
       *  and the previous token. [void - foo], [char - '*'], etc
       *
       * SS_DANGLE: - space between variable and '*' or '&' is eaten
       *     void     foo;  // gap=5
       *     char    *bar;  // gap=5, as the '*' doesn't count
       *     foomatic foo;  // gap=1
       *  The gap is the columns between 'foo' and the chunk before the first
       *  '*' or '&'. [void - foo], [char - bar], etc
       *
       * If the gap < m_gap, then the column is bumped out by the difference.
       * So, if m_gap is 2, then the above would be:
       * SS_IGNORE:
       *     void      foo;  // gap=6
       *     char *    foo;  // gap=4
       *     foomatic  foo;  // gap=2
       * SS_INCLUDE:
       *     void      foo;  // gap=6
       *     char      *foo; // gap=6
       *     foomatic  foo;  // gap=2
       * SS_DANGLE:
       *     void      foo;  // gap=6
       *     char     *bar;  // gap=6, as the '*' doesn't count
       *     foomatic  foo;  // gap=2
       * Right aligned numbers:
       *     #define A    -1
       *     #define B   631
       *     #define C     3
       * Left aligned numbers:
       *     #define A     -1
       *     #define B     631
       *     #define C     3
       *
       * In the code below, pc is set to the item that is aligned.
       * In the above examples, that is 'foo', '*', '-', or 63.
       *
       * Ref is set to the last part of the type.
       * In the above examples, that is 'void', 'char', 'foomatic', 'A', or 'B'.
       *
       * The '*' and '&' can float between the two.
       *
       * If align_on_tabstop=true, then SS_DANGLE is changed to SS_INCLUDE.
       */

      if (cpd.settings[UO_align_on_tabstop].b && (m_star_style == SS_DANGLE))
      {
         m_star_style = SS_INCLUDE;
      }

      /* Find ref. Back up to the real item that is aligned. */
      prev = start;
      while (((prev = chunk_get_prev(prev)) != NULL) &&
             (chunk_is_star(prev) ||
              chunk_is_addr(prev) ||
              (prev->type == CT_TPAREN_OPEN)))
      {
         /* do nothing - we want prev when this exits */
      }
      ref = prev;
      if (chunk_is_newline(ref))
      {
         ref = chunk_get_next(ref);
      }

      /* Find the item that we are going to align. */
      ali = start;
      if (m_star_style != SS_IGNORE)
      {
         /* back up to the first '*' preceding the token */
         prev = chunk_get_prev(ali);
         while (chunk_is_star(prev))
         {
            ali  = prev;
            prev = chunk_get_prev(ali);
         }
         if (chunk_is_token(prev, CT_TPAREN_OPEN))
         {
            ali  = prev;
            prev = chunk_get_prev(ali);
         }
      }
      if (m_amp_style != SS_IGNORE)
      {
         /* back up to the first '&' preceding the token */
         prev = chunk_get_prev(ali);
         while (chunk_is_addr(prev))
         {
            ali  = prev;
            prev = chunk_get_prev(ali);
         }
      }

      /* Tighten down the spacing between ref and start */
      if (!cpd.settings[UO_align_keep_extra_space].b)
      {
          tmp_col = ref->column;
          tmp     = ref;
          while (tmp != start)
          {
              next     = chunk_get_next(tmp);
              tmp_col += space_col_align(tmp, next);
              if (next->column != tmp_col)
              {
                  align_to_column(next, tmp_col);
              }
              tmp = next;
          }
      }

      /* Set the column adjust and gap */
      col_adj = 0;
      gap     = 0;
      if (ref != ali)
      {
         gap = ali->column - (ref->column + ref->len());
      }
      tmp = ali;
      if (chunk_is_token(tmp, CT_TPAREN_OPEN))
      {
         tmp = chunk_get_next(tmp);
      }
      if ((chunk_is_star(tmp) && (m_star_style == SS_DANGLE)) ||
          (chunk_is_addr(tmp) && (m_amp_style == SS_DANGLE)))
      {
         col_adj = start->column - ali->column;
         gap     = start->column - (ref->column + ref->len());
      }

      /* See if this pushes out the max_col */
      endcol = ali->column + col_adj;
      if (gap < m_gap)
      {
         endcol += m_gap - gap;
      }

      // LOG_FMT(LSYS, "[%p] line %d pc='%s' [%s] col:%d ali='%s' [%s] col:%d ref='%s' [%s] col:%d  col_adj=%d  endcol=%d, ss=%d as=%d, gap=%d\n",
      //         this,
      //         start->orig_line,
      //         start->str.c_str(), get_token_name(start->type), start->column,
      //         ali->str.c_str(), get_token_name(ali->type), ali->column,
      //         ref->str.c_str(), get_token_name(ref->type), ref->column,
      //         col_adj, endcol, m_star_style, m_amp_style, gap);

      ali->align.col_adj = col_adj;
      ali->align.ref     = ref;
      ali->align.start   = start;
      m_aligned.Push_Back(ali, seqnum);
      m_last_added = 1;

      LOG_FMT(LAS, "Add-[%s]: line %d, col %d, adj %d : ref=[%s] endcol=%d\n",
              ali->str.c_str(), ali->orig_line, ali->column, ali->align.col_adj,
              ref->str.c_str(), endcol);

      if (m_min_col > endcol)
      {
         m_min_col = endcol;
      }

      if (endcol > m_max_col)
      {
         LOG_FMT(LAS, "Add-aligned [%d/%d/%d]: line %d, col %d : max_col old %d, new %d - min_col %d\n",
                 seqnum, m_nl_seqnum, m_seqnum,
                 ali->orig_line, ali->column, m_max_col, endcol, m_min_col);
         m_max_col = endcol;

         /**
          * If there were any entries that were skipped, re-add them as they
          * may now be within the threshold
          */
         if (!m_skipped.Empty())
         {
            ReAddSkipped();
         }
      }
      else
      {
         LOG_FMT(LAS, "Add-aligned [%d/%d/%d]: line %d, col %d : col %d <= %d - min_col %d\n",
                 seqnum, m_nl_seqnum, m_seqnum,
                 ali->orig_line, ali->column, endcol, m_max_col, m_min_col);
      }
   }
   else
   {
      /* The threshold check failed, so add it to the skipped list */
      m_skipped.Push_Back(start, seqnum);
      m_last_added = 2;

      LOG_FMT(LAS, "Add-skipped [%d/%d/%d]: line %d, col %d <= %d + %d\n",
              seqnum, m_nl_seqnum, m_seqnum,
              start->orig_line, start->column, m_max_col, m_thresh);
   }
}
示例#7
0
static void check_bool_parens(chunk_t *popen, chunk_t *pclose, int nest)
{
   LOG_FUNC_ENTRY();

   chunk_t *ref        = popen;
   bool    hit_compare = false;

   LOG_FMT(LPARADD, "%s(%d): popen on %zu, col %zu, pclose on %zu, col %zu, level=%zu\n",
           __func__, nest,
           popen->orig_line, popen->orig_col,
           pclose->orig_line, pclose->orig_col,
           popen->level);

   chunk_t *pc = popen;
   while ((pc = chunk_get_next_ncnl(pc)) != nullptr && pc != pclose)
   {
      if (pc->flags & PCF_IN_PREPROC)
      {
         LOG_FMT(LPARADD2, " -- bail on PP %s [%s] at line %zu col %zu, level %zu\n",
                 get_token_name(pc->type),
                 pc->text(), pc->orig_line, pc->orig_col, pc->level);
         return;
      }

      if (  chunk_is_token(pc, CT_BOOL)
         || chunk_is_token(pc, CT_QUESTION)
         || chunk_is_token(pc, CT_COND_COLON)
         || chunk_is_token(pc, CT_COMMA))
      {
         LOG_FMT(LPARADD2, " -- %s [%s] at line %zu col %zu, level %zu\n",
                 get_token_name(pc->type),
                 pc->text(), pc->orig_line, pc->orig_col, pc->level);
         if (hit_compare)
         {
            hit_compare = false;
            add_parens_between(ref, pc);
         }
         ref = pc;
      }
      else if (chunk_is_token(pc, CT_COMPARE))
      {
         LOG_FMT(LPARADD2, " -- compare [%s] at line %zu col %zu, level %zu\n",
                 pc->text(), pc->orig_line, pc->orig_col, pc->level);
         hit_compare = true;
      }
      else if (chunk_is_paren_open(pc))
      {
         chunk_t *next = chunk_skip_to_match(pc);
         if (next != nullptr)
         {
            check_bool_parens(pc, next, nest + 1);
            pc = next;
         }
      }
      else if (  chunk_is_token(pc, CT_BRACE_OPEN)
              || chunk_is_token(pc, CT_SQUARE_OPEN)
              || chunk_is_token(pc, CT_ANGLE_OPEN))
      {
         // Skip [], {}, and <>
         pc = chunk_skip_to_match(pc);
      }
   }

   if (hit_compare && ref != popen)
   {
      add_parens_between(ref, pclose);
   }
} // check_bool_parens
示例#8
0
void tokenize_cleanup(void)
{
   chunk_t *pc   = chunk_get_head();
   chunk_t *prev = NULL;
   chunk_t *next;
   chunk_t *tmp;
   chunk_t *tmp2;
   bool    in_type_cast = false;

   pc   = chunk_get_head();
   next = chunk_get_next_ncnl(pc);
   while ((pc != NULL) && (next != NULL))
   {
      /* Change '[' + ']' into '[]' */
      if ((pc->type == CT_SQUARE_OPEN) && (next->type == CT_SQUARE_CLOSE))
      {
         pc->type = CT_TSQUARE;
         pc->str  = "[]";
         pc->len  = 2;
         chunk_del(next);
         pc->orig_col_end += 1;
         next              = chunk_get_next_ncnl(pc);
      }

      if ((pc->type == CT_DOT) && ((cpd.lang_flags & LANG_ALLC) != 0))
      {
         pc->type = CT_MEMBER;
      }

      /* Determine the version stuff (D only) */
      if (pc->type == CT_D_VERSION)
      {
         if (next->type == CT_PAREN_OPEN)
         {
            pc->type = CT_D_VERSION_IF;
         }
         else
         {
            if (next->type != CT_ASSIGN)
            {
               LOG_FMT(LERR, "%s:%d %s: version: Unexpected token %s\n",
                       cpd.filename, pc->orig_line, __func__, get_token_name(next->type));
               cpd.error_count++;
            }
            pc->type = CT_WORD;
         }
      }

      /* Determine the scope stuff (D only) */
      if (pc->type == CT_D_SCOPE)
      {
         if (next->type == CT_PAREN_OPEN)
         {
            pc->type = CT_D_SCOPE_IF;
         }
         else
         {
            pc->type = CT_TYPE;
         }
      }

      /**
       * Change CT_BASE before CT_PAREN_OPEN to CT_WORD.
       * public myclass() : base() {
       * }
       */
      if ((pc->type == CT_BASE) && (next->type == CT_PAREN_OPEN))
      {
         pc->type = CT_WORD;
      }

      /**
       * Change CT_WORD after CT_ENUM, CT_UNION, or CT_STRUCT to CT_TYPE
       * Change CT_WORD before CT_WORD to CT_TYPE
       */
      if (next->type == CT_WORD)
      {
         if ((pc->type == CT_ENUM) ||
             (pc->type == CT_UNION) ||
             (pc->type == CT_STRUCT))
         {
            next->type = CT_TYPE;
         }
         if (pc->type == CT_WORD)
         {
            pc->type = CT_TYPE;
         }
      }

      /* change extern to qualifier if extern isn't followed by a string or
       * an open paren
       */
      if (pc->type == CT_EXTERN)
      {
         if (next->type == CT_STRING)
         {
            /* Probably 'extern "C"' */
         }
         else if (next->type == CT_PAREN_OPEN)
         {
            /* Probably 'extern (C)' */
         }
         else
         {
            /* Something else followed by a open brace */
            tmp = chunk_get_next_ncnl(next);
            if ((tmp != NULL) || (tmp->type != CT_BRACE_OPEN))
            {
               pc->type = CT_QUALIFIER;
            }
         }
      }

      /**
       * Change CT_STAR to CT_PTR_TYPE if preceded by CT_TYPE,
       * CT_QUALIFIER, or CT_PTR_TYPE.
       */
      if ((next->type == CT_STAR) &&
          ((pc->type == CT_TYPE) ||
           (pc->type == CT_QUALIFIER) ||
           (pc->type == CT_PTR_TYPE)))
      {
         next->type = CT_PTR_TYPE;
      }

      if ((pc->type == CT_TYPE_CAST) &&
          (next->type == CT_ANGLE_OPEN))
      {
         next->parent_type = CT_TYPE_CAST;
         in_type_cast      = true;
      }

      /**
       * Change angle open/close to CT_COMPARE, if not a template thingy
       */
      if ((pc->type == CT_ANGLE_OPEN) && (pc->parent_type != CT_TYPE_CAST))
      {
         check_template(pc);
      }
      if ((pc->type == CT_ANGLE_CLOSE) && (pc->parent_type != CT_TEMPLATE))
      {
         if (in_type_cast)
         {
            in_type_cast    = false;
            pc->parent_type = CT_TYPE_CAST;
         }
         else
         {
            next = handle_double_angle_close(pc);
         }
      }

      if ((cpd.lang_flags & LANG_D) != 0)
      {
         /* Check for the D string concat symbol '~' */
         if ((pc->type == CT_INV) &&
             ((prev->type == CT_STRING) ||
              (prev->type == CT_WORD) ||
              (next->type == CT_STRING)))
         {
            pc->type = CT_CONCAT;
         }

         /* Check for the D template symbol '!' (word + '!' + word or '(') */
         if ((pc->type == CT_NOT) &&
             (prev->type == CT_WORD) &&
             ((next->type == CT_PAREN_OPEN) ||
              (next->type == CT_WORD) ||
              (next->type == CT_TYPE)))
         {
            pc->type = CT_D_TEMPLATE;
         }

         /* handle "version(unittest) { }" vs "unittest { }" */
         if ((pc->type == CT_UNITTEST) && (prev->type == CT_PAREN_OPEN))
         {
            pc->type = CT_WORD;
         }
      }

      if ((cpd.lang_flags & LANG_CPP) != 0)
      {
         /* Change Word before '::' into a type */
         if ((pc->type == CT_WORD) && (next->type == CT_DC_MEMBER))
         {
            pc->type = CT_TYPE;
         }
      }

      /* Change get/set to CT_WORD if not followed by a brace open */
      if ((pc->type == CT_GETSET) && (next->type != CT_BRACE_OPEN))
      {
         if ((next->type == CT_SEMICOLON) &&
             ((prev->type == CT_BRACE_CLOSE) ||
              (prev->type == CT_BRACE_OPEN) ||
              (prev->type == CT_SEMICOLON)))
         {
            pc->type          = CT_GETSET_EMPTY;
            next->parent_type = CT_GETSET;
         }
         else
         {
            pc->type = CT_WORD;
         }
      }

      /* Change item after operator (>=, ==, etc) to a CT_OPERATOR_VAL
       * Usually the next item is part of the operator.
       * In a few cases the next few tokens are part of it:
       *  operator +       - common case
       *  operator >>      - need to combine '>' and '>'
       *  operator ()
       *  operator []      - already converted to TSQUARE
       *  operator new []
       *  operator delete []
       *  operator const char *
       *  operator const B&
       *  operator std::allocator<U>
       *
       * In all cases except the last, this will put the entire operator value
       * in one chunk.
       */
      if (pc->type == CT_OPERATOR)
      {
         /* Handle special case of () operator -- [] already handled */
         if (next->type == CT_PAREN_OPEN)
         {
            tmp = chunk_get_next(next);
            if ((tmp != NULL) && (tmp->type == CT_PAREN_CLOSE))
            {
               next->str  = "()";
               next->len  = 2;
               next->type = CT_OPERATOR_VAL;
               chunk_del(tmp);
               next->orig_col_end += 1;
            }
         }
         else if ((next->type == CT_ANGLE_CLOSE) &&
                  (memcmp(next->str, ">>", 2) == 0))
         {
            next->len++;
            next->orig_col_end++;
            next->type = CT_OPERATOR_VAL;
            chunk_del(chunk_get_next(next));
         }
         else if (next->flags & PCF_PUNCTUATOR)
         {
            next->type = CT_OPERATOR_VAL;
         }
         else
         {
            next->type = CT_TYPE;

            /* Replace next with a collection of all tokens that are part of
             * the type.
             */
            char opbuf[256];
            int  len;

            len = snprintf(opbuf, sizeof(opbuf), "%.*s",
                           next->len, next->str);

            tmp2 = next;
            while ((tmp = chunk_get_next(tmp2)) != NULL)
            {
               if ((tmp->type != CT_WORD) &&
                   (tmp->type != CT_TYPE) &&
                   (tmp->type != CT_QUALIFIER) &&
                   (tmp->type != CT_STAR) &&
                   (tmp->type != CT_AMP) &&
                   (tmp->type != CT_TSQUARE))
               {
                  break;
               }
               /* Change tmp into a type so that space_needed() works right */
               make_type(tmp);
               len += snprintf(opbuf + len, sizeof(opbuf) - len, "%s%.*s",
                               space_needed(tmp2, tmp) ? " " : "",
                               tmp->len, tmp->str);
               tmp2 = tmp;
            }

            while ((tmp2 = chunk_get_next(next)) != tmp)
            {
               chunk_del(tmp2);
            }

            next->str    = strdup(opbuf);
            next->len    = len;
            next->flags |= PCF_OWN_STR;
            next->type   = CT_OPERATOR_VAL;

            next->orig_col_end = next->orig_col + next->len;
         }
         next->parent_type = CT_OPERATOR;

         LOG_FMT(LOPERATOR, "%s: %d:%d operator '%.*s'\n",
                 __func__, pc->orig_line, pc->orig_col,
                 next->len, next->str);
      }

      /* Change private, public, protected into either a qualifier or label */
      if (pc->type == CT_PRIVATE)
      {
         /* Handle Qt slots - maybe should just check for a CT_WORD? */
         if (chunk_is_str(next, "slots", 5) || chunk_is_str(next, "Q_SLOTS", 7))
         {
            tmp = chunk_get_next(next);
            if ((tmp != NULL) && (tmp->type == CT_COLON))
            {
               next = tmp;
            }
         }
         if (next->type == CT_COLON)
         {
            next->type = CT_PRIVATE_COLON;
            if ((tmp = chunk_get_next_ncnl(next)) != NULL)
            {
               tmp->flags |= PCF_STMT_START | PCF_EXPR_START;
            }
         }
         else
         {
            pc->type = (chunk_is_str(pc, "signals", 7) || chunk_is_str(pc, "Q_SIGNALS", 9)) ? CT_WORD : CT_QUALIFIER;
         }
      }

      /* Look for <newline> 'EXEC' 'SQL' */
      if (chunk_is_str(pc, "EXEC", 4) && chunk_is_str(next, "SQL", 3))
      {
         tmp = chunk_get_prev(pc);
         if (chunk_is_newline(tmp))
         {
            tmp = chunk_get_next(next);
            if (chunk_is_str_case(tmp, "BEGIN", 5))
            {
               pc->type = CT_SQL_BEGIN;
            }
            else if (chunk_is_str_case(tmp, "END", 3))
            {
               pc->type = CT_SQL_END;
            }
            else
            {
               pc->type = CT_SQL_EXEC;
            }

            /* Change words into CT_SQL_WORD until CT_SEMICOLON */
            while (tmp != NULL)
            {
               if (tmp->type == CT_SEMICOLON)
               {
                  break;
               }
               if ((tmp->len > 0) && isalpha(*tmp->str))
               {
                  tmp->type = CT_SQL_WORD;
               }
               tmp = chunk_get_next_ncnl(tmp);
            }
         }
      }

      /* handle MS abomination 'for each' */
      if ((pc->type == CT_FOR) && chunk_is_str(next, "each", 4) &&
          (next == chunk_get_next(pc)))
      {
         /* merge the two */
         pc->len = next->orig_col_end - pc->orig_col;
         pc->orig_col_end = next->orig_col_end;
         chunk_del(next);
         next = chunk_get_next_ncnl(pc);
         /* label the 'in' */
         if (next && (next->type == CT_PAREN_OPEN))
         {
            tmp = chunk_get_next_ncnl(next);
            while (tmp && (tmp->type != CT_PAREN_CLOSE))
            {
               if (chunk_is_str(tmp, "in", 2))
               {
                  tmp->type = CT_IN;
                  break;
               }
               tmp = chunk_get_next_ncnl(tmp);
            }
         }
      }

      /* ObjectiveC allows keywords to be used as identifiers in some situations
       * This is a dirty hack to allow some of the more common situations.
       */
      if (cpd.lang_flags & LANG_OC)
      {
         if (((pc->type == CT_IF) ||
             (pc->type == CT_FOR) ||
             (pc->type == CT_WHILE)) &&
             !chunk_is_token(next, CT_PAREN_OPEN))
         {
            pc->type = CT_WORD;
         }
         if ((pc->type == CT_DO) &&
             (chunk_is_token(prev, CT_MINUS) ||
              chunk_is_token(next, CT_SQUARE_CLOSE)))
         {
            pc->type = CT_WORD;
         }
      }

      /* Another hack to clean up more keyword abuse */
      if ((pc->type == CT_CLASS) &&
          (chunk_is_token(prev, CT_DOT) ||
           chunk_is_token(next, CT_DOT)))
      {
         pc->type = CT_WORD;
      }

      /* Detect Objective C class name */
      if ((pc->type == CT_OC_IMPL) ||
          (pc->type == CT_OC_INTF) ||
          (pc->type == CT_OC_PROTOCOL))
      {
         if (next->type != CT_PAREN_OPEN)
         {
            next->type = CT_OC_CLASS;
         }
         next->parent_type = pc->type;

         tmp = chunk_get_next_ncnl(next);
         if (tmp != NULL)
         {
            tmp->flags |= PCF_STMT_START | PCF_EXPR_START;
         }

         tmp = chunk_get_next_type(pc, CT_OC_END, pc->level);
         if (tmp != NULL)
         {
            tmp->parent_type = pc->type;
         }
      }

      if (pc->type == CT_OC_INTF)
      {
         tmp = chunk_get_next_ncnl(pc, CNAV_PREPROC);
         while ((tmp != NULL) && (tmp->type != CT_OC_END))
         {
            if (get_token_pattern_class(tmp->type) != PATCLS_NONE)
            {
               LOG_FMT(LOBJCWORD, "@interface %d:%d change '%.*s' (%s) to CT_WORD\n",
                       pc->orig_line, pc->orig_col, tmp->len, tmp->str,
                       get_token_name(tmp->type));
               tmp->type = CT_WORD;
            }
            tmp = chunk_get_next_ncnl(tmp, CNAV_PREPROC);
         }
      }

      /* Detect Objective-C categories and class extensions */
      /* @interface ClassName (CategoryName) */
      /* @implementation ClassName (CategoryName) */
      /* @interface ClassName () */
      /* @implementation ClassName () */
      if (((pc->parent_type == CT_OC_IMPL) ||
           (pc->parent_type == CT_OC_INTF) ||
           (pc->type == CT_OC_CLASS)) &&
          (next->type == CT_PAREN_OPEN))
      {
         next->parent_type = pc->parent_type;

         tmp = chunk_get_next(next);
         if ((tmp != NULL) && (tmp->next != NULL))
         {
            if (tmp->type == CT_PAREN_CLOSE)
            {
               //tmp->type        = CT_OC_CLASS_EXT;
               tmp->parent_type = pc->parent_type;
            }
            else
            {
               tmp->type        = CT_OC_CATEGORY;
               tmp->parent_type = pc->parent_type;
            }
         }

         tmp = chunk_get_next_type(pc, CT_PAREN_CLOSE, pc->level);
         if (tmp != NULL)
         {
            tmp->parent_type = pc->parent_type;
         }
      }

      /**
       * Objective C @dynamic and @synthesize
       *  @dynamic xxx, yyy;
       *  @synthesize xxx, yyy;
       * Just find the semicolon and mark it.
       */
      if (pc->type == CT_OC_DYNAMIC)
      {
         tmp = chunk_get_next_type(pc, CT_SEMICOLON, pc->level);
         if (tmp != NULL)
         {
            tmp->parent_type = pc->type;
         }
      }

      /* Detect Objective C @property
       *  @property NSString *stringProperty;
       *  @property(nonatomic, retain) NSMutableDictionary *shareWith;
       */
      if (pc->type == CT_OC_PROPERTY)
      {
         if (next->type != CT_PAREN_OPEN)
         {
            next->flags |= PCF_STMT_START | PCF_EXPR_START;
         }
         else
         {
            next->parent_type = pc->type;

            tmp = chunk_get_next_type(pc, CT_PAREN_CLOSE, pc->level);
            if (tmp != NULL)
            {
               tmp->parent_type = pc->type;
               tmp = chunk_get_next_ncnl(tmp);
               if (tmp != NULL)
               {
                  tmp->flags |= PCF_STMT_START | PCF_EXPR_START;

                  tmp = chunk_get_next_type(tmp, CT_SEMICOLON, pc->level);
                  if (tmp != NULL)
                  {
                     tmp->parent_type = pc->type;
                  }
               }
            }
         }
      }

      /* Detect Objective C @selector
       *  @selector(msgNameWithNoArg)
       *  @selector(msgNameWith1Arg:)
       *  @selector(msgNameWith2Args:arg2Name:)
       */
      if ((pc->type == CT_OC_SEL) && (next->type == CT_PAREN_OPEN))
      {
         next->parent_type = pc->type;

         tmp = chunk_get_next(next);
         if (tmp != NULL)
         {
            tmp->type        = CT_OC_SEL_NAME;
            tmp->parent_type = pc->type;

            while ((tmp = chunk_get_next_ncnl(tmp)) != NULL)
            {
               if (tmp->type == CT_PAREN_CLOSE)
               {
                  tmp->parent_type = CT_OC_SEL;
                  break;
               }
               tmp->type        = CT_OC_SEL_NAME;
               tmp->parent_type = pc->type;
            }
         }
      }

      /* Mark Objective-C blocks (aka lambdas or closures)
       *  The syntax and usage is exactly like C function pointers with two exceptions:
       *  Instead of an asterisk they have a caret as pointer symbol.
       *  In method declarations which take a block as parameter, there can be anonymous blocks, e.g.: (^)
       *  1. block literal: ^{ ... };
       *  2. block declaration: return_t (^name) (int arg1, int arg2, ...) NB: return_t is optional and name can be optional if found as param in a method declaration.
       *  3. block expression: ^ return_t (int arg) { ... }; NB: return_t is optional
       *
       *  See http://developer.apple.com/mac/library/documentation/Cocoa/Conceptual/Blocks for more info...
       */
      if ((cpd.lang_flags & LANG_OC) &&
          (pc->type == CT_ARITH) &&
          (chunk_is_str(pc, "^", 1)) &&
          (prev->type != CT_NUMBER) &&
          (prev->type != CT_NUMBER_FP))
      {
         /* mark objc blocks caret so that we can process it later*/
         pc->type = CT_OC_BLOCK_CARET;

         if (prev->type == CT_PAREN_OPEN)
         {
            /* block declaration */
            pc->parent_type = CT_OC_BLOCK_TYPE;
         }
         else if ((next->type == CT_PAREN_OPEN) ||
                  (next->type == CT_BRACE_OPEN))
         {
            /* block expression without return type */
            /* block literal */
            pc->parent_type = CT_OC_BLOCK_EXPR;
         }
         else
         {
            /* block expression with return type (seldomly used) */
            if (prev->type == CT_ASSIGN)
            {
               /* shortcut to spare the peeking below
                * the XOR operator wouldn't be used directly
                * after an assign all by itself */
               pc->parent_type = CT_OC_BLOCK_EXPR;
            }
            else
            {
               /* this ones tricky because we don't know how many
                * stars the return type has - if there even is one */
               tmp = pc;
               while ((tmp = chunk_get_next(tmp)) != NULL)
               {
                  /* we just peek ahead and see if the line contains
                   * an open brace somewhere.
                   * FIXME: this check needs to be more thorough. */
                  if (tmp->type == CT_BRACE_OPEN)
                  {
                     pc->parent_type = CT_OC_BLOCK_EXPR;
                     break;
                  }
               }
            }
         }
      }

      /* Handle special preprocessor junk */
      if (pc->type == CT_PREPROC)
      {
         pc->parent_type = next->type;
      }

      /* Detect "pragma region" and "pragma endregion" */
      if ((pc->type == CT_PP_PRAGMA) && (next->type == CT_PREPROC_BODY))
      {
         if ((memcmp(next->str, "region", 6) == 0) ||
             (memcmp(next->str, "endregion", 9) == 0))
         {
            pc->type = (*next->str == 'r') ? CT_PP_REGION : CT_PP_ENDREGION;

            prev->parent_type = pc->type;
         }
      }

      /* Check for C# nullable types '?' is in next */
      if ((cpd.lang_flags & LANG_CS) &&
          (next->type == CT_QUESTION) &&
          (next->orig_col == (pc->orig_col + pc->len)))
      {
         tmp = chunk_get_next_ncnl(next);
         if (tmp != NULL)
         {
            bool doit = ((tmp->type == CT_PAREN_CLOSE) ||
                         (tmp->type == CT_ANGLE_CLOSE));

            if (tmp->type == CT_WORD)
            {
               tmp2 = chunk_get_next_ncnl(tmp);
               if ((tmp2 != NULL) &&
                   ((tmp2->type == CT_SEMICOLON) ||
                    (tmp2->type == CT_ASSIGN) ||
                    (tmp2->type == CT_COMMA)))
               {
                  doit = true;
               }
            }

            if (doit)
            {
               pc->len++;
               chunk_del(next);
               next = tmp;
            }
         }
      }

      /* Change 'default(' into a sizeof-like statement */
      if ((cpd.lang_flags & LANG_CS) &&
          (pc->type == CT_DEFAULT) &&
          (next->type == CT_PAREN_OPEN))
      {
         pc->type = CT_SIZEOF;
      }

      if ((pc->type == CT_UNSAFE) && (next->type != CT_BRACE_OPEN))
      {
         pc->type = CT_QUALIFIER;
      }

      if ((pc->type == CT_USING) && (next->type == CT_PAREN_OPEN))
      {
         pc->type = CT_USING_STMT;
      }

      /* Add minimal support for C++0x rvalue references */
      if ((pc->type == CT_BOOL) && chunk_is_str(pc, "&&", 2))
      {
         if (prev->type == CT_TYPE)
         {
            pc->type = CT_BYREF;
         }
      }

      /* HACK: treat try followed by a colon as a qualifier to handle this:
       *   A::A(int) try : B() { } catch (...) { }
       */
      if ((pc->type == CT_TRY) && chunk_is_str(pc, "try", 3) &&
          (next != NULL) && (next->type == CT_COLON))
      {
         pc->type = CT_QUALIFIER;
      }

      /* TODO: determine other stuff here */

      prev = pc;
      pc   = next;
      next = chunk_get_next_ncnl(pc);
   }
}