const cst_string *ts_get(cst_tokenstream *ts)
{
/* Get next token */
/* Skip whitespace */
get_token_sub_part(ts,
TS_CHARCLASS_WHITESPACE,
&ts->whitespace,
&ts->ws_max);
/* quoted strings currently ignored */
ts->token_pos = ts->file_pos - 1;
/* Get prepunctuation */
if (ts->current_char != TS_EOF &&
ts_charclass(ts->current_char,TS_CHARCLASS_PREPUNCT,ts))
get_token_sub_part(ts,
TS_CHARCLASS_PREPUNCT,
&ts->prepunctuation,
&ts->prep_max);
else if (ts->prepunctuation)
ts->prepunctuation[0] = '\0';
/* Get the symbol itself */
if (ts->current_char != TS_EOF &&
ts_charclass(ts->current_char,TS_CHARCLASS_SINGLECHAR,ts))
{
if (2 >= ts->token_max) extend_buffer(&ts->token,&ts->token_max);
ts->token[0] = ts->current_char;
ts->token[1] = '\0';
ts_getc(ts);
}
else
get_token_sub_part_2(ts,
TS_CHARCLASS_WHITESPACE, /* end class1 */
&ts->token,
&ts->token_max);
/* This'll have token *plus* post punctuation in ts->token */
/* Get postpunctuation */
if (ts->p_postpunctuationsymbols[0])
get_token_postpunctuation(ts);
return ts->token;
}
const cst_string *ts_get_quoted_token(cst_tokenstream *ts,
char quote,
char escape)
{
/* for reading the next quoted token that starts with quote and
ends with quote, quote may appear only if preceded by escape */
int l, p;
/* Hmm can't change quotes within a ts */
ts->charclass[(unsigned int)quote] |= TS_CHARCLASS_QUOTE;
ts->charclass[(unsigned int)escape] |= TS_CHARCLASS_QUOTE;
/* skipping whitespace */
get_token_sub_part(ts,TS_CHARCLASS_WHITESPACE,
&ts->whitespace,
&ts->ws_max);
ts->token_pos = ts->file_pos - 1;
#import "OpenEarsStaticAnalysisToggle.h"
#ifdef STATICANALYZEDEPENDENCIES
#define __clang_analyzer__ 1
#endif
#if !defined(__clang_analyzer__) || defined(STATICANALYZEDEPENDENCIES)
#undef __clang_analyzer__
if (ts->current_char == quote)
{ /* go until quote */
ts_getc(ts);
l=0;
for (p=0; ((ts->current_char != TS_EOF) &&
(ts->current_char != quote));
p++)
{
if (p >= ts->token_max)
extend_buffer(&ts->token,&ts->token_max);
ts->token[p] = ts->current_char;
ts_getc(ts);
if (ts->current_char == escape)
{
ts_get(ts);
if (p >= ts->token_max)
extend_buffer(&ts->token,&ts->token_max);
ts->token[p] = ts->current_char;
ts_get(ts);
}
}
ts->token[p] = '\0';
ts_getc(ts);
}
#endif
else /* its not quotes, like to be careful dont you */
{ /* treat is as standard token */
/* Get prepunctuation */
get_token_sub_part(ts,TS_CHARCLASS_PREPUNCT,
&ts->prepunctuation,
&ts->prep_max);
/* Get the symbol itself */
if (!ts_charclass(ts->current_char,TS_CHARCLASS_SINGLECHAR,ts))
{
if (2 >= ts->token_max) extend_buffer(&ts->token,&ts->token_max);
ts->token[0] = ts->current_char;
ts->token[1] = '\0';
ts_getc(ts);
}
else
get_token_sub_part_2(ts,
TS_CHARCLASS_WHITESPACE, /* end class1 */
&ts->token,
&ts->token_max);
/* This'll have token *plus* post punctuation in ts->token */
/* Get postpunctuation */
get_token_postpunctuation(ts);
}
return ts->token;
}
const unsigned char *ts_get_quoted_token(cst_tokenstream *ts,
char quote,
char escape)
{
/* for reading the next quoted token that starts with quote and
ends with quote, quote may appear only if preceded by escape */
int l;
/* Hmm can't change quotes within a ts */
ts->charclass[(unsigned int)quote] |= TS_CHARCLASS_QUOTE;
ts->charclass[(unsigned int)escape] |= TS_CHARCLASS_QUOTE;
/* skipping whitespace */
get_token_sub_part(ts,TS_CHARCLASS_WHITESPACE,
&ts->whitespace,
&ts->ws_max);
ts->token_pos = ts->file_pos - 1;
if (ts->current_char == quote)
{ /* go until quote */
ts_getc(ts);
l=0;
while (!ts_eof(ts))
{
get_token_sub_part_2(ts,TS_CHARCLASS_QUOTE,
&ts->token,&ts->token_max);
if (ts->current_char == escape)
{
ts_getc(ts);
l = cst_strlen(ts->token);
if (l+1 >= ts->token_max)
extend_buffer(&ts->token,&ts->token_max);
ts->token[l] = ts->current_char;
ts->token[l+1] = '\0';
ts_getc(ts);
}
else
break;
}
ts_getc(ts);
}
else /* its not quotes, like to be careful dont you */
{ /* treat is as standard token */
/* Get prepunctuation */
get_token_sub_part(ts,TS_CHARCLASS_PREPUNCT,
&ts->prepunctuation,
&ts->prep_max);
/* Get the symbol itself */
if (!ts_charclass(ts->current_char,TS_CHARCLASS_SINGLECHAR,ts))
{
if (2 >= ts->token_max) extend_buffer(&ts->token,&ts->token_max);
ts->token[0] = ts->current_char;
ts->token[1] = '\0';
ts_getc(ts);
}
else
get_token_sub_part_2(ts,
TS_CHARCLASS_WHITESPACE, /* end class1 */
&ts->token,
&ts->token_max);
/* This'll have token *plus* post punctuation in ts->token */
/* Get postpunctuation */
get_token_postpunctuation(ts);
}
return ts->token;
}