void
nsa_token(struct nsa_parser *p, enum nsa_ptypes type, void *ref, const char *s)
{
struct nsa_token *t = new_token();
if (type == NSA_P_STOP)
{
t->type = NSA_T_STOP;
}
else
{
unsigned char *s2 = npool_copy((const unsigned char *)s,p->pool), *brack;
t->type = NSA_T_GRAPHEME;
grapheme(t) = new_grapheme();
grapheme_overt(t) = 1;
grapheme_text_ref(t) = new_text_ref();
grapheme_text_ref(t)->ptype = type;
switch (type)
{
case NSA_P_LEMM:
grapheme_text_ref(t)->t.lemmptr = ref;
break;
case NSA_P_LITERAL:
grapheme_text_ref(t)->t.literal = ref;
break;
case NSA_P_LINK:
grapheme_text_ref(t)->t.linkptr = ref;
break;
default:
break;
}
if ((brack = (unsigned char *)strchr((const char *)s2,'('))
&& (isdigit(*s2) || ((brack-s2)==1 && (*s2 == 'n' || *s2 == 'N'))))
{
grapheme_num(t) = (char *)s2;
*brack++ = '\0';
grapheme_unit(t) = (char *)brack;
while (*brack && ')' != *brack)
++brack;
*brack = '\0';
}
}
list_add(p->toks,t);
}
static void
lem_index(struct location8 *l8, const char *field, const char *toks, ...)
{
struct sn_tab *snp = statnames(field,strlen(field));
if (snp)
{
va_list ap;
const char *tok;
int saved_start_column = start_column;
BIT_SET(start_column,snp->uid);
va_start(ap,toks);
tok = toks;
while (1)
{
int ntok = 0;
if (tok)
{
char *tmp = malloc(strlen(tok)+1), *end;
static char manglebuf[128];
end = strcpy(tmp,tok);
start_column = saved_start_column|snp->uid;
if (ntok++)
grapheme_boundary(' ');
while (1)
{
char *t = end;
int punct = 0;
while (*end && !strchr(". -=,:;", *end))
++end;
if (*end)
{
punct = *end;
*end++ = '\0';
}
if (debug_flag)
fprintf(debug_f, "%s %s\n", debug_label, t);
grapheme((char*)keymangler((unsigned char*)t,KM_FOLD,manglebuf,128));
if (punct)
{
grapheme_boundary('-');
#if 0
if ('=' == punct)
grapheme_decr_start_column();
#endif
punct = 0;
}
if (!*end)
break;
}
tok = va_arg(ap,const char *);
}
else
break;
}
va_end(ap);
BIT_CLR(start_column,snp->uid);
start_column = saved_start_column;
}