Esempio n. 1
0
void
nsa_token(struct nsa_parser *p, enum nsa_ptypes type, void *ref, const char *s)
{
  struct nsa_token *t = new_token();

  if (type == NSA_P_STOP)
    {
      t->type = NSA_T_STOP;
    }
  else
    {
      unsigned char *s2 = npool_copy((const unsigned char *)s,p->pool), *brack;
      t->type = NSA_T_GRAPHEME;
      grapheme(t) = new_grapheme();
      grapheme_overt(t) = 1;
      grapheme_text_ref(t) = new_text_ref();
      grapheme_text_ref(t)->ptype = type;
      switch (type)
	{
	case NSA_P_LEMM:
	  grapheme_text_ref(t)->t.lemmptr = ref;
	  break;
	case NSA_P_LITERAL:
	  grapheme_text_ref(t)->t.literal = ref;
	  break;
	case NSA_P_LINK:
	  grapheme_text_ref(t)->t.linkptr = ref;
	  break;
	default:
	  break;
	}
      if ((brack = (unsigned char *)strchr((const char *)s2,'('))
	  && (isdigit(*s2) || ((brack-s2)==1 && (*s2 == 'n' || *s2 == 'N'))))
	{
	  grapheme_num(t) = (char *)s2;
	  *brack++ = '\0';
	  grapheme_unit(t) = (char *)brack;
	  while (*brack && ')' != *brack)
	    ++brack;
	  *brack = '\0';
	}
    }
  list_add(p->toks,t);
}
Esempio n. 2
0
static void
lem_index(struct location8 *l8, const char *field, const char *toks, ...)
{
  struct sn_tab *snp = statnames(field,strlen(field));
  if (snp)
    {
      va_list ap;
      const char *tok;
      int saved_start_column = start_column;

      BIT_SET(start_column,snp->uid);
      va_start(ap,toks);
      tok = toks;
      while (1)
	{
	  int ntok = 0;
	  if (tok)
	    {
	      char *tmp = malloc(strlen(tok)+1), *end;
	      static char manglebuf[128];
	      end = strcpy(tmp,tok);
	      start_column = saved_start_column|snp->uid;
	      if (ntok++)
		grapheme_boundary(' ');

	      while (1)
		{
		  char *t = end;
		  int punct = 0;

		  while (*end && !strchr(". -=,:;", *end))
		    ++end;

		  if (*end)
		    {
		      punct = *end;
		      *end++ = '\0';
		    }

		  if (debug_flag)
		    fprintf(debug_f, "%s %s\n", debug_label, t);

		  grapheme((char*)keymangler((unsigned char*)t,KM_FOLD,manglebuf,128));

		  if (punct)
		    {
		      grapheme_boundary('-');
#if 0
		      if ('=' == punct)
			grapheme_decr_start_column();
#endif
		      punct = 0;
		    }
		  
		  if (!*end)
		    break;
		}
	      tok = va_arg(ap,const char *);
	    }
	  else
	    break;
	}
      va_end(ap);
      BIT_CLR(start_column,snp->uid);
      start_column = saved_start_column;
    }