static void replace_stdout()
{
#if RUBY_VERSION_CODE >= 180
rb_io_binmode(rb_stdout); /* for mswin32 */
rb_stdout = rb_str_new("", 0);
rb_define_singleton_method(rb_stdout, "write", defout_write, 1);
rb_define_singleton_method(rb_stdout, "cancel", defout_cancel, 0);
#else
rb_defout = rb_str_new("", 0);
rb_io_binmode(rb_stdout); /* for mswin32 */
rb_define_singleton_method(rb_defout, "write", defout_write, 1);
rb_define_singleton_method(rb_defout, "cancel", defout_cancel, 0);
#endif
}
VALUE Wikitext_parser_flush(VALUE self,VALUE file){
rb_io_t * file_p_struct;
FILE * file_p;
Check_Type(file,T_FILE);
rb_io_binmode(file);
GetOpenFile(file,file_p_struct);
file_p = rb_io_stdio_file(file_p_struct);
fflush(file_p);
return Qnil;
}
// for testing and debugging only
VALUE Wikitext_parser_tokenize(VALUE self, VALUE string, VALUE file1, VALUE file2,
VALUE doc_id_v)
{
if (NIL_P(string))
return Qnil;
int doc_id = FIX2INT(doc_id_v);
rb_io_t * file_p_struct;
Check_Type(file1,T_FILE);
rb_io_binmode(file1);
GetOpenFile(file1,file_p_struct);
rb_io_check_writable(file_p_struct);
FILE * file_p1 = rb_io_stdio_file(file_p_struct);
Check_Type(file2,T_FILE);
rb_io_binmode(file2);
GetOpenFile(file2,file_p_struct);
rb_io_check_writable(file_p_struct);
FILE * file_p2 = rb_io_stdio_file(file_p_struct);
string = StringValue(string);
VALUE tokens = rb_ary_new();
char *p = RSTRING_PTR(string);
long len = RSTRING_LEN(string);
char *pe = p + len;
const char *type = NULL;
int state = DEFAULT;
int ts[1000];
int ss[1000];
int ts_size = 1 ;
int ss_size = 1 ;
ss[0] = 0;
ts[0] = 0;
token_t token;
token_t stack1[1000]; //tail, TODO add checks
token_t stack2[1000]; //head, TODO add checks
int s1_size = 0;
int s2_size = 0;
int start = 1;
do
{
//printf("%i %i\n",s1_size,s2_size);
if(start) {
next_token(&token, NULL, p, pe, ts, ss, &ts_size, &ss_size, file_p1, doc_id);
start = 0;
} else {
next_token(&token, &token, NULL, pe, ts, ss, &ts_size, &ss_size, file_p1, doc_id);
}
//rb_funcall(rb_mKernel, rb_intern("puts"), 1, wiki_token(&token));
if(token.type == END_OF_FILE){
if(state == POST_LINK) {
finish_link1(&s1_size,&s2_size,stack1,stack2,file_p1,file_p2,doc_id);
}
break;
}
switch(token.type) {
case SEPARATOR :
memcpy(stack2,stack1,sizeof(token_t)*s1_size);
s2_size = s1_size;
s1_size = 0;
state = SEPARATOR;
break;
case LINK_START :
state = LINK;
break;
case LINK_END :
if(state != SEPARATOR) {
memcpy(stack2,stack1,sizeof(token_t)*s1_size);
s2_size = s1_size;
}
state = POST_LINK;
break;
case EXT_LINK_START :
state = EXT_LINK;
break;
case EXT_LINK_END :
// TODO print what should be printed
s1_size = 0;
s2_size = 0;
break;
case ALNUM :
if(state == POST_LINK){
finish_link(&s1_size,&s2_size,stack1,stack2,file_p1,file_p2,doc_id,&token);
state = DEFAULT;
} else {
if(s1_size < 1000)
memcpy(&(stack1[s1_size++]),&token,sizeof(token_t));
else
printf("[%i]Token stack overflow\n",doc_id);
}
break;
case SPACE :
type = space_type;
case PRINTABLE :
case DEFAULT :
if(type == NULL) type = printable_type;
case NUM :
if(type == NULL) type = num_type;
if(state == POST_LINK) {
finish_link1(&s1_size,&s2_size,stack1,stack2,file_p1,file_p2,doc_id);
wikitext_print_token(&token,file_p1,doc_id,type);
state = DEFAULT;
} else {
if(s1_size < 1000)
memcpy(&(stack1[s1_size++]),&token,sizeof(token_t));
else
printf("[%i]Token stack overflow\n",doc_id);
}
break;
case CRLF :
if(state == POST_LINK){
finish_link1(&s1_size,&s2_size,stack1,stack2,file_p1,file_p2,doc_id);
wikitext_print_crlf(&token,file_p1,doc_id);
state = DEFAULT;
} else {
if(s1_size < 1000)
memcpy(&(stack1[s1_size++]),&token,sizeof(token_t));
else
printf("[%i]Token stack overflow\n",doc_id);
}
break;
case SKIP :
if(state == POST_LINK){
finish_link1(&s1_size,&s2_size,stack1,stack2,file_p1,file_p2,doc_id);
state = DEFAULT;
}
break;
}
type = NULL;
} while(1);
return tokens;
}
