static FrList *string_to_List(const char *&input, const char *)
{
FrList *list, *prev ;
FrObject *curr ;
list = prev = 0 ;
input++ ; // consume initial left parenthesis
FramepaC_read_nesting_level++ ;
char c ;
while ((c = FrSkipWhitespace(input)) != ')' && c != '\0')
{
FrObject *obj = string_to_FrObject(input) ;
if (obj == symbolPERIOD && FrSkipWhitespace(input) != ')')
{
// period is not last item, so check if it's a dotted pair
curr = string_to_FrObject(input) ;
// was period second-to-last in list?
if ((c = FrSkipWhitespace(input)) == ')')
{
if (!list)
prev = list = new FrList(0) ;
prev->replacd(curr) ;
break ;
}
else
{
if (!list)
prev = list = new FrList(obj) ;
else
{
obj = new FrList(obj) ;
prev->replacd(obj) ;
prev = (FrList*)obj ;
}
obj = curr ;
}
}
curr = new FrList(obj) ;
if (!list)
list = (FrList*)curr ;
else
prev->replacd(curr) ;
prev = (FrList*)curr ;
}
if (c != ')')
expected_right_paren(list) ;
else
input++ ;
if (--FramepaC_read_nesting_level <= 0 && FramepaC_read_associations)
{
FramepaC_read_associations->freeObject() ;
FramepaC_read_associations = 0 ;
}
return list ;
}
static FrObject *string_to_Queue(const char *&input, const char *)
{
FrObject *result = string_to_FrObject(input) ;
if (result && result->consp())
return new FrQueue((FrList*)result) ;
else
return new FrQueue(0) ;
}
static void string_to_Facet(const char *&input,FrFrame *frame,FrSymbol *slot)
{
FrSymbol *facet ;
input++ ; // consume the left bracket
facet = string_to_Symbol(input) ; // get facet name
if (!facet || !facet->symbolp()) // the name must be a symbol
{
FrWarning(errmsg_facet_symbol) ;
return ;
}
frame->createFacet(slot,facet) ;
char c ;
while ((c = FrSkipWhitespace(input)) != ']' && c != '\0')
{
frame->addFillerNoCopy(slot,facet,string_to_FrObject(input)) ;
}
if (c != ']')
FrWarning(errmsg_facet_malformed) ;
else
input++ ;
}
bool FrTFIDF::load(const char *filename)
{
if (filename && *filename)
{
FrITextFile wt(filename) ;
if (!wt.good())
{
FrWarningVA("unable to open term weights file '%s'",filename) ;
return false ;
}
delete ht ;
ht = new FrSymHashTable ;
FrSymbol *symEOF = FrSymbolTable::add("*EOF*") ;
char *line = wt.getline() ;
bool expanded = false ;
if (line && strncmp(line,"!!! ",4) == 0)
{
char *end = 0 ;
total_docs = (size_t)strtol(line+4,&end,10) ;
if (end && end != line+4)
{
char *tmp = end ;
size_t vocab_size = (size_t)strtol(tmp,&end,10) ;
if (vocab_size > 0 && end && end != tmp)
{
ht->expand(vocab_size+100) ;
expanded = true ;
}
}
}
if (!expanded) // ensure some reasonable starting size
ht->expand(5000) ;
while ((line = wt.getline()) != 0)
{
if (FrSkipWhitespace(line) == ';' || *line == '\0')
continue ;
const char *origline = line ;
FrSymbol *term = (FrSymbol*)string_to_FrObject(line) ;
if (term == symEOF || !term || !term->symbolp())
{
FrWarning("invalid line in term-weights file") ;
free_object(term) ;
continue ;
}
char *end = 0 ;
size_t term_freq = strtol(line,&end,10) ;
if (end && end != line)
{
line = end ;
size_t doc_freq = strtol(line,&end,10) ;
if (end != line)
{
if (doc_freq > 0 && term_freq > 0)
{
FrSymHashEntry *entry = tfidfRecord(term) ;
FrTFIDFrecord *rec = new FrTFIDFrecord(term_freq,doc_freq) ;
if (entry)
{
delete (FrTFIDFrecord*)entry->getUserData() ;
entry->setUserData(rec) ;
}
else
ht->add(term,(void*)rec) ;
continue ;
}
FrWarning("invalid data in term-weights file -- both term\n"
"\tand document frequencies must be nonzero") ;
free_object(term) ;
continue ;
}
}
FrWarningVA("expected two integers following the term '%s'; line was\n"
"\t%s", term->symbolName(), origline) ;
free_object(term) ;
}
return true ;
}
return false ;
}
