value_t fl_accum_julia_symbol(value_t *args, u_int32_t nargs)
{
argcount("accum-julia-symbol", nargs, 2);
ios_t *s = fl_toiostream(args[1], "accum-julia-symbol");
if (!iscprim(args[0]) || ((cprim_t*)ptr(args[0]))->type != wchartype)
type_error("accum-julia-symbol", "wchar", args[0]);
uint32_t wc = *(uint32_t*)cp_data((cprim_t*)ptr(args[0]));
ios_t str;
ios_mem(&str, 0);
while (jl_id_char(wc)) {
ios_getutf8(s, &wc);
if (wc == '!') {
uint32_t nwc;
ios_peekutf8(s, &nwc);
// make sure != is always an operator
if (nwc == '=') {
ios_ungetc('!', s);
break;
}
}
ios_pututf8(&str, wc);
if (ios_peekutf8(s, &wc) == IOS_EOF)
break;
}
ios_pututf8(&str, 0);
return symbol(normalize(str.buf));
}
value_t fl_accum_julia_symbol(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
{
argcount(fl_ctx, "accum-julia-symbol", nargs, 2);
ios_t *s = fl_toiostream(fl_ctx, args[1], "accum-julia-symbol");
if (!iscprim(args[0]) || ((cprim_t*)ptr(args[0]))->type != fl_ctx->wchartype)
type_error(fl_ctx, "accum-julia-symbol", "wchar", args[0]);
uint32_t wc = *(uint32_t*)cp_data((cprim_t*)ptr(args[0]));
ios_t str;
int allascii=1;
ios_mem(&str, 0);
do {
allascii &= (wc <= 0x7f);
ios_getutf8(s, &wc);
if (wc == '!') {
uint32_t nwc;
ios_peekutf8(s, &nwc);
// make sure != is always an operator
if (nwc == '=') {
ios_ungetc('!', s);
break;
}
}
ios_pututf8(&str, wc);
if (ios_peekutf8(s, &wc) == IOS_EOF)
break;
} while (jl_id_char(wc));
ios_pututf8(&str, 0);
return symbol(fl_ctx, allascii ? str.buf : normalize(fl_ctx, str.buf));
}
value_t fl_skipws(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
{
argcount(fl_ctx, "skip-ws", nargs, 2);
ios_t *s = fl_toiostream(fl_ctx, args[0], "skip-ws");
int newlines = (args[1]!=fl_ctx->F);
uint32_t wc=0;
value_t skipped = fl_ctx->F;
while (1) {
if (ios_peekutf8(s, &wc) == IOS_EOF) {
ios_getutf8(s, &wc); // to set EOF flag if this is a true EOF
if (!ios_eof(s))
lerror(fl_ctx, symbol(fl_ctx, "error"), "incomplete character");
return fl_ctx->FL_EOF;
}
if (!ios_eof(s) && (is_uws(wc) || is_bom(wc)) && (newlines || wc!=10)) {
skipped = fl_ctx->T;
ios_getutf8(s, &wc);
}
else {
break;
}
}
return skipped;
}
value_t fl_skipws(value_t *args, u_int32_t nargs)
{
argcount("skip-ws", nargs, 2);
ios_t *s = fl_toiostream(args[0], "skip-ws");
int newlines = (args[1]!=FL_F);
uint32_t wc=0;
if (ios_peekutf8(s, &wc) == IOS_EOF)
return FL_EOF;
value_t skipped = FL_F;
while (!ios_eof(s) && (is_uws(wc) || is_bom(wc)) && (newlines || wc!=10)) {
skipped = FL_T;
ios_getutf8(s, &wc);
ios_peekutf8(s, &wc);
}
return skipped;
}
DLLEXPORT uint32_t jl_getutf8(ios_t *s)
{
uint32_t wc=0;
ios_getutf8(s, &wc);
return wc;
}
static u_int32_t peek(void)
{
char c, *end;
fixnum_t x;
int ch, base;
if (toktype != TOK_NONE)
return toktype;
c = nextchar();
if (ios_eof(F)) return TOK_NONE;
if (c == '(') {
toktype = TOK_OPEN;
}
else if (c == ')') {
toktype = TOK_CLOSE;
}
else if (c == '[') {
toktype = TOK_OPENB;
}
else if (c == ']') {
toktype = TOK_CLOSEB;
}
else if (c == '\'') {
toktype = TOK_QUOTE;
}
else if (c == '`') {
toktype = TOK_BQ;
}
else if (c == '"') {
toktype = TOK_DOUBLEQUOTE;
}
else if (c == '#') {
ch = ios_getc(F); c = (char)ch;
if (ch == IOS_EOF)
lerror(ParseError, "read: invalid read macro");
if (c == '.') {
toktype = TOK_SHARPDOT;
}
else if (c == '\'') {
toktype = TOK_SHARPQUOTE;
}
else if (c == '\\') {
uint32_t cval;
if (ios_getutf8(F, &cval) == IOS_EOF)
lerror(ParseError, "read: end of input in character constant");
if (cval == (uint32_t)'u' || cval == (uint32_t)'U' ||
cval == (uint32_t)'x') {
read_token('u', 0);
if (buf[1] != '\0') { // not a solitary 'u','U','x'
if (!read_numtok(&buf[1], &tokval, 16))
lerror(ParseError,
"read: invalid hex character constant");
cval = numval(tokval);
}
}
else if (cval >= 'a' && cval <= 'z') {
read_token((char)cval, 0);
tokval = symbol(buf);
if (buf[1] == '\0') /* one character */;
else if (tokval == nulsym) cval = 0x00;
else if (tokval == alarmsym) cval = 0x07;
else if (tokval == backspacesym) cval = 0x08;
else if (tokval == tabsym) cval = 0x09;
else if (tokval == linefeedsym) cval = 0x0A;
else if (tokval == newlinesym) cval = 0x0A;
else if (tokval == vtabsym) cval = 0x0B;
else if (tokval == pagesym) cval = 0x0C;
else if (tokval == returnsym) cval = 0x0D;
else if (tokval == escsym) cval = 0x1B;
else if (tokval == spacesym) cval = 0x20;
else if (tokval == deletesym) cval = 0x7F;
else
lerrorf(ParseError, "read: unknown character #\\%s", buf);
}
toktype = TOK_NUM;
tokval = mk_wchar(cval);
}
else if (c == '(') {
toktype = TOK_SHARPOPEN;
}
else if (c == '<') {
lerror(ParseError, "read: unreadable object");
}
else if (isdigit(c)) {
read_token(c, 1);
c = (char)ios_getc(F);
if (c == '#')
toktype = TOK_BACKREF;
else if (c == '=')
toktype = TOK_LABEL;
else
lerror(ParseError, "read: invalid label");
errno = 0;
x = strtol(buf, &end, 10);
if (*end != '\0' || errno)
lerror(ParseError, "read: invalid label");
tokval = fixnum(x);
}
else if (c == '!') {
// #! single line comment for shbang script support
do {
ch = ios_getc(F);
} while (ch != IOS_EOF && (char)ch != '\n');
return peek();
}
else if (c == '|') {
// multiline comment
int commentlevel=1;
while (1) {
ch = ios_getc(F);
hashpipe_gotc:
if (ch == IOS_EOF)
lerror(ParseError, "read: eof within comment");
if ((char)ch == '|') {
ch = ios_getc(F);
if ((char)ch == '#') {
commentlevel--;
if (commentlevel == 0)
break;
else
continue;
}
goto hashpipe_gotc;
}
else if ((char)ch == '#') {
ch = ios_getc(F);
if ((char)ch == '|')
commentlevel++;
else
goto hashpipe_gotc;
}
}
// this was whitespace, so keep peeking
return peek();
}
else if (c == ';') {
// datum comment
(void)do_read_sexpr(UNBOUND); // skip
return peek();
}
else if (c == ':') {
// gensym
ch = ios_getc(F);
if ((char)ch == 'g')
ch = ios_getc(F);
read_token((char)ch, 0);
errno = 0;
x = strtol(buf, &end, 10);
if (*end != '\0' || buf[0] == '\0' || errno)
lerror(ParseError, "read: invalid gensym label");
toktype = TOK_GENSYM;
tokval = fixnum(x);
}
else if (symchar(c)) {
read_token(ch, 0);
if (((c == 'b' && (base= 2)) ||
(c == 'o' && (base= 8)) ||
(c == 'd' && (base=10)) ||
(c == 'x' && (base=16))) &&
(isdigit_base(buf[1],base) ||
buf[1]=='-')) {
if (!read_numtok(&buf[1], &tokval, base))
lerrorf(ParseError, "read: invalid base %d constant", base);
return (toktype=TOK_NUM);
}
toktype = TOK_SHARPSYM;
tokval = symbol(buf);
}
else {
lerror(ParseError, "read: unknown read macro");
}
}
else if (c == ',') {
toktype = TOK_COMMA;
ch = ios_getc(F);
if (ch == IOS_EOF)
return toktype;
if ((char)ch == '@')
toktype = TOK_COMMAAT;
else if ((char)ch == '.')
toktype = TOK_COMMADOT;
else
ios_ungetc((char)ch, F);
}
else {
if (!read_token(c, 0)) {
if (buf[0]=='.' && buf[1]=='\0') {
return (toktype=TOK_DOT);
}
else {
if (read_numtok(buf, &tokval, 0))
return (toktype=TOK_NUM);
}
}
toktype = TOK_SYM;
tokval = symbol(buf);
}
return toktype;
}