int ScriptHandler::checkClickstr(const char *buf, bool recursive_flag)
{
    if ( buf[0] == '@' || buf[0] == '\\' ) return 1;

    if (clickstr_list == NULL) return 0;

    bool double_byte_check = true;
    char *click_buf = clickstr_list;
    while(click_buf[0]){
#ifdef ENABLE_1BYTE_CHAR
        if (click_buf[0] == '`'){
            click_buf++;
            double_byte_check = false;
            continue;
        }
#endif
        if (double_byte_check){
            if ( click_buf[0] == buf[0] && click_buf[1] == buf[1] ){
                if (!recursive_flag && checkClickstr(buf+2, true) > 0) return 0;
                return 2;
            }
            click_buf += 2;
        }
        else{
            if ( click_buf[0] == buf[0] ){
                if (!recursive_flag && checkClickstr(buf+1, true) > 0) return 0;
                return 1;
            }
            click_buf++;
        }
    }

    return 0;
}
int ScriptHandler::checkClickstr(const char* buf, bool recursive_flag)
{
    if (buf[0] == '@' || buf[0] == '\\') return 1;
    int bytes;
    wchar c = file_encoding->DecodeChar(buf, bytes);
    if (clickstr_list.find(c) != clickstr_list.end()) {
	if (!recursive_flag && checkClickstr(buf + bytes, true)) return 0;
	return bytes;
    }
    return 0;
}
// basic parser function
const char* ScriptHandler::readToken(bool no_kidoku)
{
    current_script = next_script;
    const char* buf = current_script;
    end_status = END_NONE;
    current_variable.type = VAR_NONE;

    text_flag = false;

    SKIP_SPACE(buf);
    if (!no_kidoku) markAsKidoku(buf);
    
readTokenTop:
    string_buffer.trunc(0);
    char ch = *buf;
    if (ch == ';') { // comment
        addStrBuf(ch);
        do {
            ch = *++buf;
            addStrBuf(ch);
        } while (ch != 0x0a && ch != '\0');
    }
    else if (ch & 0x80
             || (ch >= '0' && ch <= '9')
             || ch == '@' || ch == '\\' || ch == '/'
             || ch == '%' || ch == '?' || ch == '$'
             || ch == '[' || ch == '('
             || ch == '!' || ch == '#' || ch == ',' || ch == '"') {
        // text
        if (ch != '!' and !warned_unmarked) {
//            errorWarning("unmarked text found"); //Mion: stop warnings, for compatibility
            // TODO: make this more robust; permit only !-directives
//            warned_unmarked = true;
        }
        bool loop_flag = true;
        bool ignore_click_flag = false;
        do {
            char bytes = file_encoding->NextCharSize(buf);
            if (bytes > 1) {
                if (textgosub_flag && !ignore_click_flag && checkClickstr(buf))
                    loop_flag = false;
                string_buffer.add(buf, bytes);
                buf += bytes;
                SKIP_SPACE(buf);
                ch = *buf;
            }
            else {
                if (ch == '%' || ch == '?') {
                    addIntVariable(&buf);
                }
                else if (ch == '$') {
                    addStrVariable(&buf);
                }
                else {
                    if (textgosub_flag && !ignore_click_flag &&
                        checkClickstr(buf))
                        loop_flag = false;

                    string_buffer += ch;
                    buf++;
                    ignore_click_flag = false;
                    if (ch == '_') ignore_click_flag = true;
                }

                // CHECKME: why do we ignore text markers here?
                if (isadigit(ch) &&
                    (isawspace(*buf) || *buf == file_encoding->TextMarker()) &&
                    (string_buffer.length() % 2)) {
                    string_buffer += ' ';
                }

                ch = *buf;
                if (ch == 0x0a || ch == '\0' || !loop_flag ||
                    ch == file_encoding->TextMarker()) {
                    break;
                }

                SKIP_SPACE(buf);
                ch = *buf;
            }
        }
        while (ch != 0x0a && ch != '\0' && loop_flag &&
               ch != file_encoding->TextMarker()) /*nop*/;
        if (loop_flag && ch == 0x0a && !(textgosub_flag && linepage_flag)) {
            string_buffer += ch;
            if (!no_kidoku) markAsKidoku(buf++);
        }

        text_flag = true;
    }
    else if (ch == file_encoding->TextMarker()) {
        ch = *++buf;
        while (ch != file_encoding->TextMarker() && ch != 0x0a && ch != '\0') {
            if ((ch == '\\' || ch == '@') &&
                (textgosub_flag || buf[1] == 0x0a || buf[1] == 0)) {
                string_buffer += *buf++;
                ch = *buf;
                break;
            }

            // Interpolate expressions.
            if (ch == '{' &&
                (buf[1] == '%' || buf[1] == '$' || buf[1] == '?'))
            {
                const char* start = buf + 1;
                while (*buf && *buf != '\n' && *buf != '}') ++buf;
                if (*buf != '}')
                    errorAndExit("interpolation missing }");
                pstring var_expr(start, buf++ - start);
                const char* var_iter = var_expr;
                if (var_expr[0] == '$') {
                    pstring val = parseStr(&var_iter);
                    if (val[0] == file_encoding->TextMarker()) val.remove(0, 1);
                    string_buffer += val;
                }
                else {
                    string_buffer += stringFromInteger(parseInt(&var_iter),
                                                       -1);
                }   
                ch = *buf;
                continue;
            }
            
            if (file_encoding->UseTags() && ch == '~' && (ch = *++buf) != '~') {
                while (ch != '~') {
                    int l;
                    string_buffer += file_encoding->TranslateTag(buf, l);
                    buf += l;
                    ch = *buf;
                }
                ch = *++buf;
                continue;
            }

            int bytes;
            // NOTE: we don't substitute ligatures at this stage.
            string_buffer += file_encoding->Encode(file_encoding->DecodeChar(buf, bytes));
            buf += bytes;
            ch = *buf;
        }
        if (ch == file_encoding->TextMarker() && !textgosub_flag) ++buf;

        if (ch == 0x0a && !(textgosub_flag && linepage_flag)) {
            string_buffer += ch;
            if (!no_kidoku) markAsKidoku(buf++);
        }

        text_flag   = true;
    }
    else if ((ch >= 'a' && ch <= 'z')
             || (ch >= 'A' && ch <= 'Z')
             || ch == '_') { // command
        do {
            if (ch >= 'A' && ch <= 'Z') ch += 'a' - 'A';

            string_buffer += ch;
            ch = *++buf;
        }
        while ((ch >= 'a' && ch <= 'z')
               || (ch >= 'A' && ch <= 'Z')
               || (ch >= '0' && ch <= '9')
               || ch == '_');
    }
    else if (ch == '*') { // label
        return readLabel();
    }
    else if (ch == '~' || ch == 0x0a || ch == ':') {
        string_buffer += ch;
        if (!no_kidoku) markAsKidoku(buf++);
    }
    else if (ch != '\0') {
        fprintf(stderr, "readToken: skip unknown heading character %c (%x)\n",
		ch, ch);
        buf++;
        goto readTokenTop;
    }

    if (text_flag)
        next_script = buf;
    else
        next_script = checkComma(buf);

    return string_buffer;
}
// basic parser function
const char *ScriptHandler::readToken()
{
    current_script = next_script;
    wait_script = NULL;
    char *buf = current_script;
    end_status = END_NONE;
    current_variable.type = VAR_NONE;

    text_flag = false;

    SKIP_SPACE( buf );
    markAsKidoku( buf );

  readTokenTop:
    string_counter = 0;
    char ch = *buf;
    if (ch == ';'){ // comment
        addStringBuffer( ch );
        do{
            ch = *++buf;
            addStringBuffer( ch );
        } while ( ch != 0x0a && ch != '\0' );
    }
    else if (ch & 0x80 ||
             (ch >= '0' && ch <= '9') ||
             ch == '@' || ch == '\\' || ch == '/' ||
             ch == '%' || ch == '?' || ch == '$' ||
             ch == '[' || ch == '(' || ch == '<' ||
#ifndef ENABLE_1BYTE_CHAR
             ch == '`' ||
#endif             
             (!english_mode && ch == '>') ||
             ch == '!' || ch == '#' || ch == ',' || ch == '"'){ // text
        bool ignore_clickstr_flag = false;
        while(1){
            if ( IS_TWO_BYTE(ch) ){
                addStringBuffer( ch );
                ch = *++buf;
                if (ch == 0x0a || ch == '\0') break;
                addStringBuffer( ch );
                buf++;
                if (!wait_script && !ignore_clickstr_flag &&
                    checkClickstr(buf-2) > 0)
                    wait_script = buf;
                ignore_clickstr_flag = false;
            }
            else{
                ignore_clickstr_flag = false;
                if (ch == '%' || ch == '?'){
                    addIntVariable(&buf);
                    SKIP_SPACE(buf);
                }
                else if (ch == '$'){
                    addStrVariable(&buf);
                    SKIP_SPACE(buf);
                }
                else{
                    if (ch == 0x0a || ch == '\0') break;
                    addStringBuffer( ch );
                    buf++;
                    if (ch == '_') ignore_clickstr_flag = true;
                    if (!wait_script && ch == '@') wait_script = buf;
                }
            }
            ch = *buf;
        }

        text_flag = true;
    }
#ifdef ENABLE_1BYTE_CHAR
    else if (ch == '`'){
        ch = *++buf;
        while (ch != '`' && ch != 0x0a && ch !='\0'){
            if ( IS_TWO_BYTE(ch) ){
                addStringBuffer( ch );
                ch = *++buf;
            }
            addStringBuffer( ch );
            ch = *++buf;
        }
        if (ch == '`') buf++;
        
        text_flag = true;
        end_status |= END_1BYTE_CHAR;
    }
#endif
    else if (english_mode && ch == '>'){
        ch = *++buf;
        while (1){
            if (ch == 0x0a || ch =='\0') break;

            if (ch != '\t') 
                addStringBuffer( ch );
            ch = *++buf;
        }
        
        text_flag = true;
        end_status |= END_1BYTE_CHAR;
    }
    else if ((ch >= 'a' && ch <= 'z') || 
             (ch >= 'A' && ch <= 'Z') ||
             ch == '_'){ // command
        do{
            if (ch >= 'A' && ch <= 'Z') ch += 'a' - 'A';
            addStringBuffer( ch );
            ch = *++buf;
        }
        while((ch >= 'a' && ch <= 'z') || 
              (ch >= 'A' && ch <= 'Z') ||
              (ch >= '0' && ch <= '9') ||
              ch == '_');
    }
    else if (ch == '*'){ // label
        return readLabel();
    }
    else if (ch == '~' || ch == 0x0a || ch == ':'){
        addStringBuffer( ch );
        markAsKidoku( buf++ );
    }
    else if (ch != '\0'){
        fprintf(stderr, "readToken: skip unknown heading character %c (%x)\n", ch, ch);
        buf++;
        goto readTokenTop;
    }

    next_script = checkComma(buf);

    //printf("readToken [%s] len=%d [%c(%x)] %p\n", string_buffer, strlen(string_buffer), ch, ch, next_script);

    return string_buffer;
}