/* this doesn't read the peculiar identifiers, they are scanned in the
* main reader body */
static object read_identifier(FILE *in)
{
object o;
int str_len = 0;
int str_size = STRING_MIN_BUFFER;
char *buffer;
int c;
buffer = xmalloc(str_size);
c = tolower(fgetc(in));
assert(is_initial(c));
buffer[str_len++] = c;
while (1) {
c = fgetc(in);
if (is_delimiter(c)) {
ungetc(c, in);
break;
}
if (!is_subsequent(c))
error("Symbol has bad name -- read", nil);
/* we're a lower case scheme */
buffer[str_len++] = tolower(c);
}
o = make_symbol(buffer, str_len);
xfree(buffer);
return o;
}
static void do_state_init(char *ibuf, char **pscan)
{
if (**pscan == '#'){
stack_push(&state_stack, current_state);
current_state = STATE_SHARP;
++*pscan;
} else if (**pscan == '\''){
stack_push(&state_stack, current_state);
current_state = STATE_QUOTE;
++*pscan;
do_state_init(ibuf, pscan);
}else if (**pscan == '.'){
stack_push(&state_stack, current_state);
current_state = STATE_DOT;
++*pscan;
} else if (**pscan == '+'){
stack_push(&state_stack, current_state);
current_state = STATE_ADD;
++*pscan;
} else if (**pscan == '-'){
stack_push(&state_stack, current_state);
current_state = STATE_SUB;
++*pscan;
} else if (is_digit(**pscan)){
stack_push(&state_stack, current_state);
current_state = STATE_NUM;
} else if (is_initial(**pscan)){
stack_push(&state_stack, current_state);
current_state = STATE_SYM;
} else if (**pscan == '"'){
stack_push(&state_stack, current_state);
current_state = STATE_STR;
++*pscan;
} else if (**pscan == '('){
parse_list(ibuf, pscan);
} else if (**pscan == ')'){
if (current_state == STATE_LIST){
parse_list(ibuf, pscan);
}else if (current_state == STATE_VEC){
parse_vector(ibuf, pscan);
}else {
printf("Error: unexpected ')' -- READ\n");
do_input_error(pscan);
}
} else if(is_atmosphere(**pscan)){
eat_atmosphere(pscan);
} else{
perror("Error: Bad input. -- READ\n");
do_input_error(pscan);
}
}
static scheme_object* read_symbol(vm* context, FILE* in)
{
string_t* str = string_create();
int c = getc(in);
while(is_initial(c) || isdigit(c) || c == '+' || c == '-')
{
string_append_char(str, c);
c = getc(in);
}
if(!is_delimiter(c))
{
fprintf(stderr, "symbol not folowed by delimiter\n");
exit(1);
}
ungetc(c, in);
scheme_object* rval = make_symbol(context, string_cstring(str));
string_free(str);
return rval;
}
Q3ListViewItem* LinguisticaMainWindow::GetMiniLexiconSubTree(
Q3ListViewItem* parent, int index)
{
CMiniLexicon* lexicon = m_lexicon->GetMiniLexicon(index);
if (lexicon == 0)
// nothing to draw
return 0;
enum eAffixLocation affixLocation = lexicon->GetAffixLocation();
// Items are inserted into the tree in reverse order
// respective to their group. They are organized by depth here,
// i.e. what is connected to the root is the first group,
// then do the groups of items connected to this first group, etc.
// The only items that get their own name are those that have children.
Q3ListViewItem* LexiconItem =
// owned by parent
new CTreeViewItem(parent,
QString(
m_lexicon->GetActiveMiniIndex() == index
? "Mini-Lexicon %1\t**ACTIVE**"
: "Mini-Lexicon %1")
.arg(index+1),
MINI_LEXICON, index);
LexiconItem->setSelectable(false);
LexiconItem->setOpen(true);
// START CONNECTED TO LexiconItem
if (lexicon->GetFSA() != 0)
static_cast<void>(new CTreeViewItem(
LexiconItem, "FSA", FSA_DOC, index));
// XXX. HMMLexemes, HMM Corpus Tokens?
if (LexiconItem != 0)
if (m_lexicon->GetDLHistory()->count() > 0)
static_cast<void>(new CTreeViewItem(
LexiconItem,
"Description length",
DESCRIPTION_LENGTH));
// Suffixes or prefixes
Q3ListViewItem* AffixesItem = 0;
if (is_initial(affixLocation)) {
// XXX. update to match suffix case
const int count = lexicon->GetPrefixes()->GetCount();
if (count > 0) {
AffixesItem = new CTreeViewItem(LexiconItem,
QString("Prefixes %1").arg(
IntToStringWithCommas(count)),
PREFIXES,
index);
AffixesItem->setOpen( true );
}
} else {
const int count = lexicon->GetSuffixes()->GetCount();
const int usecount = lexicon->GetSuffixes()->GetTotalUseCount();
lexicon->GetSuffixes()->RecomputeCorpusCount();
const int corpuscount = lexicon->GetSuffixes()->GetCorpusCount();
if (count > 0) {
AffixesItem = new CTreeViewItem(LexiconItem,
QString("Suffixes %1 : %2 : %3").arg(
IntToStringWithCommas(count),
IntToStringWithCommas(usecount),
IntToStringWithCommas(corpuscount)),
SUFFIXES,
index);
AffixesItem->setOpen(true);
}
}
{
int dummy;
const int count =
lexicon->GetWords()->HowManyAreAnalyzed(dummy, this);
if (count > 0) {
static_cast<void>(new CTreeViewItem(LexiconItem,
QString("Analyzed words %1").arg(
IntToStringWithCommas(count)),
ANALYZED_WORDS,
index));
}
}
Q3ListViewItem* WordsItem = 0;
{
const int count = lexicon->GetWords()->GetCount();
if (count > 0) {
QString caption = QString("Words %1 Z: %2")
.arg(IntToStringWithCommas(count))
.arg(lexicon->GetWords()->GetZ_Local());
WordsItem = new CTreeViewItem(LexiconItem,
caption, WORDS, index);
WordsItem->setOpen(true);
}
}
// END CONNECTED TO LexiconItem
// START CONNECTED TO WordsItem
{
const int count = lexicon->GetWords()->GetTrie()->GetCount();
if (count > 0) {
if (lexicon->GetWords()->GetReverseTrie() != 0) {
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Reverse trie %1")
.arg(IntToStringWithCommas(count)),
REVERSE_TRIE, index));
}
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Forward trie %1")
.arg(IntToStringWithCommas(count)),
TRIE, index));
}
}
// XXX. lexicon->GetVowels()?
{
const int tier1_bigrams = lexicon->GetWords()
->GetPhonologicalContentTier1Bigrams();
// XXX. probably a typo.
if (tier1_bigrams != 0) {
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Unigram description length %1")
.arg(IntToStringWithCommas(
lexicon->GetWords()
->GetPhonologicalContentUnigrams())),
UNIGRAM_INFORMATION, index));
}
if (tier1_bigrams != 0) {
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Bigram description length %1")
.arg(IntToStringWithCommas(tier1_bigrams)),
BIGRAM_INFORMATION, index));
}
}
if (const int distant_log_pr =
lexicon->GetWords()->GetDistantMI_Plog())
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Distant MI model plog: %1")
.arg(IntToStringWithCommas(distant_log_pr)),
BIGRAM_INFORMATION, index));
if (const int local_log_pr =
lexicon->GetWords()->GetLocalMI_Plog())
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Local MI model plog: %1")
.arg(IntToStringWithCommas(local_log_pr)),
BIGRAM_INFORMATION, index));
if (const double distant_denom =
lexicon->GetWords()->GetZ_Distant())
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Z (distant model): %1")
.arg(distant_denom),
BIGRAM_INFORMATION, index));
if (const double local_denom =
lexicon->GetWords()->GetZ_Local())
static_cast<void>(new CTreeViewItem(
WordsItem,
QString("Z (local) : %1")
.arg(local_denom),
BIGRAM_INFORMATION, index));
// Phones inside words
// Tier 2
if (CPhoneCollection* phones = lexicon->GetWords()->GetPhones_Tier2()) {
const int count = phones->GetCount();
const int totalcount = phones->GetCorpusCount();
if (count > 0) {
Q3ListViewItem* Tier2Item =
// owned by WordsItem
new CTreeViewItem(WordsItem,
"Tier 2 ",
PHONES_Tier2, index);
Tier2Item->setOpen(true);
static_cast<void>(new CTreeViewItem(Tier2Item,
QString("Phones %1 : %2")
.arg(IntToStringWithCommas(count),
IntToStringWithCommas(totalcount)),
PHONES_Tier2, index));
// Tier 2 biphones
CBiphoneCollection* biphones = phones->GetMyBiphones();
const int biphone_count = biphones->count();
const int total_biphone_count = biphones->m_TotalCount;
if (biphone_count > 0) {
static_cast<void>(new CTreeViewItem(
Tier2Item,
QString("Biphones %1 : %2")
.arg(IntToStringWithCommas(biphone_count),
IntToStringWithCommas(total_biphone_count)),
BIPHONES_Tier2, index));
if (const int local_score = lexicon->GetWords()
->GetTier2_LocalMI_Score())
static_cast<void>(new CTreeViewItem(
Tier2Item,
QString("Tier 2 Local MI score %1")
.arg(IntToStringWithCommas(
local_score)),
TIER2MI, index));
if (const int distant_score = lexicon->GetWords()
->GetTier2_DistantMI_Score())
static_cast<void>(new CTreeViewItem(
Tier2Item,
QString("Tier 2 Distant MI score %1")
.arg(IntToStringWithCommas(
distant_score)),
TIER2MI, index));
}
}
}
// Tier 1 Skeleton
{
CPhoneCollection* phones =
lexicon->GetWords()->GetPhones_Tier1_Skeleton();
const int count = phones->GetCount();
const int totalcount = phones->GetCorpusCount();
if (count > 0) {
Q3ListViewItem* Tier1Item =
// owned by WordsItem
new CTreeViewItem(WordsItem,
"Tier 1 Skeleton ",
PHONES_Tier1_Skeleton, index);
Tier1Item->setOpen(true);
static_cast<void>(new CTreeViewItem(
Tier1Item,
QString("Phones (skeleton) %1 : %2")
.arg(IntToStringWithCommas(count),
IntToStringWithCommas(totalcount)),
PHONES_Tier1_Skeleton, index));
CBiphoneCollection* biphones = phones->GetMyBiphones();
const int biphone_count = biphones->count();
const int biphone_totalcount = biphones->m_TotalCount;
if (biphone_count > 0) {
static_cast<void>(new CTreeViewItem(
Tier1Item,
QString("Biphones (skeleton) %1 : %2")
.arg(IntToStringWithCommas(biphone_count),
IntToStringWithCommas(biphone_totalcount)),
BIPHONES_Tier1_Skeleton, index));
}
}
}
{
// Tier 1
CPhoneCollection* phones = lexicon->GetWords()->GetPhones();
const int count = phones->GetCount();
const int totalcount = phones->GetCorpusCount();
if (count > 0) {
Q3ListViewItem* Tier1Item =
// owned by WordsItem
new CTreeViewItem(WordsItem,
"Tier 1 ",
PHONES, index);
Tier1Item->setOpen(true);
static_cast<void>(new CTreeViewItem(
Tier1Item,
QString("Phones %1 : %2")
.arg(IntToStringWithCommas(count),
IntToStringWithCommas(totalcount)),
PHONES, index));
CBiphoneCollection* biphones = phones->GetMyBiphones();
const int biphone_count = biphones->count();
const int biphone_totalcount = biphones->m_TotalCount;
if (biphone_count > 0) {
static_cast<void>(new CTreeViewItem(
Tier1Item,
QString("Biphones %1 : %2")
.arg(IntToStringWithCommas(biphone_count),
IntToStringWithCommas(biphone_totalcount)),
BIPHONES, index));
}
}
}
// END CONNECTED TO WordsItem
// START CONNECTED TO AffixesItem
{
const int count = lexicon->GetStems()->GetCount();
if (count > 0) {
static_cast<void>(new CTreeViewItem(
AffixesItem,
QString("Stems %1")
.arg(IntToStringWithCommas(count)),
STEMS, index));
}
}
{
const int count = lexicon->GetSignatures()->GetCount();
if (count > 0) {
static_cast<void>(new CTreeViewItem(
AffixesItem,
QString("Signatures %1")
.arg(IntToStringWithCommas(count)),
SIGNATURES, index));
}
}
if (LxPoSCollection* parts_of_speech = lexicon->GetPOS()) {
const int count = parts_of_speech->count();
static_cast<void>(new CTreeViewItem(AffixesItem,
QString("Parts of speech %1")
.arg(IntToStringWithCommas(count)),
POS, index));
}
return LexiconItem;
}
static boolean is_subsequent(char ch)
{
return is_initial(ch) || is_digit(ch) || is_special_subsequent(ch);
}
scheme_object* read(vm* context, FILE* in)
{
int c;
eat_whitespace(in);
c = getc(in);
if(isdigit(c) || (c == '-' && (isdigit(peek(in)))))
{
short sign = 1;//positive
long num = 0;
//read a number
if(c == '-')
{
sign = -1;
}
else
{
ungetc(c, in);
}
while(isdigit(c = getc(in)))
{
num = num *10 + (c -'0');
}
num *= sign;
if(is_delimiter(c))
{
ungetc(c, in);
return make_number(context, num);
}
//its not followed by a delimiter
fprintf(stderr, "Number not followed by a delimiter!\n");
exit(1);
}
else if(c == '#')
{//character or boolean
c = getc(in);
switch(c)
{
case 't':
return (scheme_object*) true;
case 'f':
return (scheme_object*) false;
case '\\':
return read_character(context, in);
default:
fprintf(stderr, "unknown boolean or character literal\n");
exit(1);
}
}
else if(c =='"')
{
return read_string(context, in);
}
else if(c == '(')
{
return read_pair(context, in);
}
else if(is_initial(c) || ((c == '+' || c == '-') && is_delimiter(peek(in))))
{
ungetc(c, in);
return read_symbol(context, in);
}
else if(c == '\'')
{
return cons(context, (scheme_object*) quote_symbol, cons(context, read(context, in), the_empty_list));
}
else if(c == EOF)
{
return NULL;
}
else
{
fprintf(stderr, "Unexpected input: '%c'\n", c);
exit(1);
}
fprintf(stderr, "read: illegal state\n");
exit(1);
}
static int is_subsequent(int c)
{
return is_initial(c) || isdigit(c) ||
is_special_subsequent(c);
}
object lisp_read(FILE *in)
{
int c;
while ((c = fgetc(in)) != EOF) {
/* atmosphere */
if (isspace(c)) {
continue;
}
else if (c == ';') {
while (c != EOF && c != '\n')
c = fgetc(in);
continue;
}
/* characters, booleans or numbers with radix */
else if (c == '#') {
c = fgetc(in);
switch (c) {
/* number prefixes */
case 'b':
case 'B':
case 'o':
case 'O':
case 'x':
case 'X':
case 'd':
case 'D':
case 'e':
case 'i':
ungetc(c, in);
return read_number(in);
/* booleans */
case 't':
case 'T':
return the_truth;
case 'f':
case 'F':
return the_falsity;
/* characters */
case '\\':
return read_character(in);
/* vectors */
case '(':
ungetc(c, in);
return read_vector(in);
/* commented form, read and discard */
case ';':
lisp_read(in);
continue;
case '<':
error("Object cannot be read back -- read", nil);
default:
error("Unexpected character -- read", nil);
}
}
/* number */
else if (isdigit(c) ||
((c == '-') && isdigit(peek_char(in))) ||
((c == '+') && isdigit(peek_char(in)))) {
ungetc(c, in);
return read_number(in);
}
/* string */
else if (c == '"') {
return read_string(in);
}
/* symbol */
else if (is_initial(c)) {
ungetc(c, in);
return read_identifier(in);
}
/* peculiar identifiers */
else if (((c == '+') || c == '-') && is_delimiter(peek_char(in))) {
return make_symbol_c((c == '+' ? "+" : "-"));
}
/* stuff starting with dot, FIXME for floats */
else if (c == '.') {
if (is_delimiter(peek_char(in)))
error("Illegal use of . -- read", nil);
c = fgetc(in);
if (c != '.' || peek_char(in) != '.')
error("Symbol has bad name -- read", nil);
c = fgetc(in);
if (!is_delimiter(peek_char(in)))
error("Symbol has bad name -- read", nil);
return _ellipsis;
}
/* pair */
else if (c == '(') {
return read_pair(in);
}
/* quote */
else if (c == '\'') {
return cons(_quote, cons(lisp_read(in), nil));
}
/* quasiquote */
else if (c == '`') {
return cons(_quasiquote, cons(lisp_read(in), nil));
}
/* unquote & unquote-splicing */
else if (c == ',') {
if (peek_char(in) == '@') {
c = fgetc(in);
return cons(_unquote_splicing, cons(lisp_read(in), nil));
} else
return cons(_unquote, cons(lisp_read(in), nil));
}
else
error("Unexpected character -- read", nil);
}
return end_of_file;
}
