uint64_t
pack_key(char *key, int len) {
uint64_t code = 0;
for (int i = 0; i < len; i++) {
code = push_char(code, *key);
key = next_char(key);
}
return code;
}
void lex_init (char* filename, int maxlen) {
inputname = filename;
input = fopen(filename, "r");
//Get the lexer into a usable state for the parser
curln = 1;
buffer = malloc(maxlen);
next_char();
next();
}
static bool
match_input (const char *s, bool consume)
{
int n; /* number of characters matched */
int ch; /* input character */
const char *t;
bool result = false;
ch = peek_input ();
if (ch != to_uchar (*s))
return false; /* fail */
if (s[1] == '\0')
{
if (consume)
next_char ();
return true; /* short match */
}
next_char ();
for (n = 1, t = s++; peek_input () == to_uchar (*s++); )
{
next_char ();
n++;
if (*s == '\0') /* long match */
{
if (consume)
return true;
result = true;
break;
}
}
/* Failed or shouldn't consume, push back input. */
{
struct obstack *h = push_string_init ();
/* `obstack_grow' may be macro evaluating its arg 1 several times. */
obstack_grow (h, t, n);
}
push_string_finish ();
return result;
}
void skip_to_next_line(struct tml_stream *stream)
{
for (;;) {
int ch = peek_char(stream);
next_char(stream);
if (ch == '\n' || ch == '\r' || ch == -1)
return;
}
}
FAXPP_Error
xml_decl_version_value_apos_state(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case '\'':
env->state = xml_decl_encoding_ws_state;
token_end_position(env);
report_token(XML_DECL_VERSION_TOKEN, env);
next_char(env);
break;
LINE_ENDINGS
default:
next_char(env);
return UNKNOWN_XML_VERSION;
}
return NO_ERROR;
}
FAXPP_Error
xml_decl_seen_question_state(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case '>':
env->state = initial_misc_state;
report_empty_token(XML_DECL_END_TOKEN, env);
next_char(env);
token_start_position(env);
break;
LINE_ENDINGS
default:
next_char(env);
return INVALID_CHAR_IN_XML_DECL;
}
return NO_ERROR;
}
/* sare peste spatii si comentarii stil C++ */
void sari_spatiu()
{
char c = curent_char();
do
{
while ( isspace (c) ) c = next_char();
if ( c == '/' )
{
c = view_next_char();
if ( c == '/' )
{
while ( c!='\n') c = next_char();
c = next_char();
}
else break;
}
}while( ( isspace(c) || curent_char() == '/')
&& pozitie_in_fisier <= marime_fisier);
}
FAXPP_Error
elementdecl_end_state(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case '>':
env->nesting_level -= 1;
base_state(env);
report_empty_token(ELEMENTDECL_END_TOKEN, env);
break;
LINE_ENDINGS
default:
next_char(env);
return INVALID_ELEMENTDECL_CONTENT;
}
next_char(env);
return NO_ERROR;
}
/*
* Skip whitespace characters
*/
void
_lex::_skip_whitespace(void)
{
while(isspace(get_char()))
if(!next_char())
break;
_skip_comment();
}
void BankRBTree::runRecommend(string id, string oid, int len, RecommendId& rid, int degree_c, int degree_a) {
if (degree_c == 0 && degree_a == 0) {
string now_id = id;
DataNode la = DataNode(&now_id, NULL);
DataNode* res = (DataNode*)rb_find(rb_tree, &la);
if (res == NULL) {
// it means that the id isn't exist in the bank
rid.push_back(id);
return;
}
}
string tmpid;
int delta = abs_num((int)oid.length() - (int)id.length());
if (degree_a > delta) {
if (id.length() >= oid.length() && id.length() < MAX_STRING_SIZE) {
for (char c = '0'; ; c = next_char(c)) {
runRecommend(id + c, oid, id.length() + 1, rid, degree_c, degree_a - delta - 1);
if (c == 'z') break;
}
}
if (id.length() <= oid.length() && id.length() > len && id.length() > 1) {
tmpid = id;
tmpid.pop_back();
runRecommend(tmpid, oid, len, rid, degree_c, degree_a - delta - 1);
}
}
if (degree_c > 0) {
for (int i = max_num(min_num(id.length(), oid.length()) - degree_c, len); i < min_num(oid.length(), id.length()); i++) {
if (degree_c >= min_num(oid.length(), id.length()) - i) {
for (char c = '0'; c <= 'z'; c = next_char(c)) {
if (c != oid[i]) {
tmpid = id;
tmpid[i] = c;
runRecommend(tmpid, oid, i + 1, rid, degree_c - (min_num(oid.length(), id.length()) - i), degree_a);
}
if (c == 'z') break;
}
}
}
}
}
void Scanner::skip_comment() {
if (!is_comment_start()) {
return ;
}
char c = next_char();
if (c == '/') {
while (cur_char() != '\n' && cur_char() != 0) {
go_ahead();
}
go_ahead();
} else {
go_ahead();
go_ahead();
while (cur_char() != '*' || next_char() != '/') {
go_ahead();
}
go_ahead();
go_ahead();
}
}
angelic_buffer*
next_line(angelic_http_request_parser* parser){
angelic_buffer* buffer = angelic_buffer_malloc(100);
uint8_t c;
while((c = next_char(parser)) != '\n'){
angelic_buffer_putc(buffer, c);
}
return buffer;
}
void
PhraseParser::read_word_token() {
while (duct::CHAR_EOF != m_curchar) {
if (s_set_whitespace.contains(m_curchar)) {
break;
} else {
m_token.buffer().push_back(m_curchar);
}
next_char();
}
}
FAXPP_Error
xml_decl_question_state(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
WHITESPACE:
next_char(env);
break;
case '?':
env->state = xml_decl_seen_question_state;
token_start_position(env);
next_char(env);
break;
default:
next_char(env);
return INVALID_CHAR_IN_XML_DECL;
}
return NO_ERROR;
}
FAXPP_Error
xml_decl_standalone_value_apos_state(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case '\'':
env->stored_state = xml_decl_question_state;
env->state = ws_state;
token_end_position(env);
report_token(XML_DECL_STANDALONE_TOKEN, env);
next_char(env);
break;
LINE_ENDINGS
default:
next_char(env);
return INVALID_CHAR_IN_XML_DECL;
}
return NO_ERROR;
}
unsigned
ToneAlarm::next_dots()
{
unsigned dots = 0;
while (next_char() == '.') {
_next++;
dots++;
}
return dots;
}
void
gfc_error_recovery (void)
{
char c, delim;
if (gfc_at_eof ())
return;
for (;;)
{
c = gfc_next_char ();
if (c == '\n' || c == ';')
break;
if (c != '\'' && c != '"')
{
if (gfc_at_eof ())
break;
continue;
}
delim = c;
for (;;)
{
c = next_char ();
if (c == delim)
break;
if (c == '\n')
return;
if (c == '\\')
{
c = next_char ();
if (c == '\n')
return;
}
}
if (gfc_at_eof ())
break;
}
}
Token Algorithm::get_basic_latin_word()
{
int len = 1;
int start, end;
// Skip pre-word whitespaces and punctuations
while (m_pos < m_text_length)
{
if (len > 1)
break;
if (isalnum(m_text[m_pos]))
break;
m_pos++;
len = next_char();
}
start = m_pos;
while (m_pos < m_text_length)
{
if (len > 1)
break;
if (!isalnum(m_text[m_pos]))
break;
m_pos++;
len = next_char();
}
end = m_pos;
// Skip post-word whitespaces and punctuations
while (m_pos < m_text_length)
{
if (len > 1)
break;
if (isalnum(m_text[m_pos]))
break;
m_pos++;
len = next_char();
}
return Token(m_text+start, end-start);
}
Token *run_addop() {
char cur = next_char();
switch(cur) {
case '+':
return token_new(TOK_ADDOP, ADDOP_ADD);
case '-':
return token_new(TOK_ADDOP, ADDOP_SUBTRACT);
default:
return token_new(TOK_BLOCKED, 0);
}
}
tokenizer_t mk_tokenizer(char *filename){
tokenizer_t t = (tokenizer_t)malloc(sizeof(struct tokenizer));
FILE *fp = fopen(filename, "rb");
if(fp == NULL){printf("can't open file\n");exit(1);}
t->line_buf = mk_char_buf();
t->token_buf = mk_char_buf();
t->fp = fp;
t->c = next_char(t);
t->line = 1;
t->num = 0;
return tokenize(t);
}
FAXPP_Error
xml_decl_standalone_value_state2(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case 'y':
env->state = xml_decl_standalone_yes_state1;
break;
case 'n':
env->state = xml_decl_standalone_no_state;
break;
LINE_ENDINGS
default:
retrieve_state(env);
next_char(env);
return INVALID_CHAR_IN_XML_DECL;
}
next_char(env);
return NO_ERROR;
}
bool lexer::consume_next_if(char c) {
if (current == end) {
return false;
}
if (next_char() == c) {
return true;
}
move_back();
return false;
}
_nc_panic_mode(char ch)
{
int c;
for (;;) {
c = next_char();
if (c == ch)
return;
if (c == EOF)
return;
}
}
FAXPP_Error
elementdecl_pcdata_name_state2(FAXPP_TokenizerEnv *env)
{
while(1) {
read_char(env);
switch(env->current_char) {
WHITESPACE:
case '%':
env->state = elementdecl_pcdata_end_or_names_ws_state2;
token_end_position(env);
report_token(ELEMENTDECL_NAME_TOKEN, env);
// No next_char
return NO_ERROR;
case ')':
case '|':
env->state = elementdecl_pcdata_end_or_names_state2;
token_end_position(env);
report_token(ELEMENTDECL_NAME_TOKEN, env);
// No next_char
return NO_ERROR;
case ':':
env->state = elementdecl_pcdata_name_seen_colon_state1;
token_end_position(env);
report_token(ELEMENTDECL_PREFIX_TOKEN, env);
next_char(env);
token_start_position(env);
return NO_ERROR;
default:
break;
}
next_char(env);
if((FAXPP_char_flags(env->current_char) & env->ncname_char) == 0)
return INVALID_CHAR_IN_ELEMENTDECL_NAME;
}
// Never happens
return NO_ERROR;
}
STATIC bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) {
bool had_physical_newline = false;
while (!is_end(lex)) {
if (is_physical_newline(lex)) {
if (stop_at_newline && lex->nested_bracket_level == 0) {
break;
}
had_physical_newline = true;
next_char(lex);
} else if (is_whitespace(lex)) {
next_char(lex);
} else if (is_char(lex, '#')) {
next_char(lex);
while (!is_end(lex) && !is_physical_newline(lex)) {
next_char(lex);
}
// had_physical_newline will be set on next loop
} else if (is_char_and(lex, '\\', '\n')) {
// line-continuation, so don't set had_physical_newline
next_char(lex);
next_char(lex);
} else {
break;
}
}
return had_physical_newline;
}
/**
* gts_file_getc :
* @f: a #GtsFile.
*
* Returns: the next character in @f or EOF if the end of the file is
* reached or if an error occured.
*/
gint gts_file_getc (GtsFile * f)
{
gint c;
g_return_val_if_fail (f != NULL, EOF);
if (f->type == GTS_ERROR)
return EOF;
c = next_char (f);
f->curpos++;
while (char_in_string (c, f->comments)) {
while (c != EOF && c != '\n')
c = next_char (f);
if (c == '\n') {
f->curline++;
f->curpos = 1;
c = next_char (f);
}
}
switch (c) {
case '\n':
f->curline++;
f->curpos = 1;
break;
case '{':
f->scope++;
break;
case '}':
if (f->scope == 0) {
f->line = f->curline;
f->pos = f->curpos - 1;
gts_file_error (f, "no matching opening brace");
c = EOF;
}
else
f->scope--;
}
return c;
}
/*
* match will check to see if pattern can successfully be applied to
* the beginning of string.
*/
static int
match(register const char *pattern, register const char *string, int depth)
{
#ifdef DEBUG
printf("%smatch(\"%s\", \"%s\")\n", TABS, pattern, string);
#endif
int ch;
const char *cp;
wchar_t __nlh_char[1];
while ((ch = next_char(pattern, &cp)) != '\0')
{
const char *laststr = string;
register int testchar = (int)CHARADV(string);
switch (ch)
{
case '*': {
pattern = cp; /* skip over '*' */
string = laststr; /* reverse - testchar not used */
const char *s = string;
do
if (match(pattern, s, depth + 1))
return RETURN(1, depth);
while (CHARADV(s) != '\0');
return RETURN(0, depth);
}
case '?':
break;
case '[': {
int mt = match_class(pattern, testchar);
if (mt == 0)
return RETURN(0, depth);
else if (mt == 2 && ch != testchar)
return RETURN(0, depth);
break;
}
default:
if ((ch & ~QUOTE) != testchar)
return RETURN(0, depth);
break;
}
if (testchar == '\0')
string = laststr; // reverse string
pattern = next_patt(pattern);
}
return RETURN(CHARAT(string) == '\0', depth);
}
static void
skip_comment_line (void)
{
char c;
do
{
c = next_char ();
}
while (c != '\n');
gfc_advance_line ();
}
FAXPP_Error
elementdecl_pcdata_end_or_names_state2(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case ')':
env->elemdecl_content_level -= 1;
env->state = elementdecl_pcdata_star_state;
report_empty_token(ELEMENTDECL_RPAR_TOKEN, env);
break;
case '|':
env->state = elementdecl_pcdata_name_ws_state;
report_empty_token(ELEMENTDECL_BAR_TOKEN, env);
break;
default:
next_char(env);
return INVALID_ELEMENTDECL_CONTENT;
}
next_char(env);
return NO_ERROR;
}
FAXPP_Error
elementdecl_cp_name_state1(FAXPP_TokenizerEnv *env)
{
read_char(env);
switch(env->current_char) {
case '(':
env->elemdecl_content_level += 1;
env->state = elementdecl_cp_name_ws_state;
report_empty_token(ELEMENTDECL_LPAR_TOKEN, env);
next_char(env);
break;
LINE_ENDINGS
default:
env->state = elementdecl_cp_name_state2;
next_char(env);
if((FAXPP_char_flags(env->current_char) & env->ncname_start_char) == 0)
return INVALID_CHAR_IN_ELEMENTDECL_NAME;
break;
}
return NO_ERROR;
}
