uint32_t *FrAugmentWordIDArray(const FrWordIDList *list, uint32_t *prevIDs,
size_t &num_words, uint32_t eor,
bool reverse)
{
size_t new_words = FrCountWordIDs(list) ;
// as we may have trouble requesting a memory block for the full count
// right off the bat, work around the issue by first allocating a
// smallish block and then resizing it
uint32_t *IDs = prevIDs ;
if (!IDs && num_words + new_words > 100000)
IDs = (uint32_t*)FrMalloc(FrMaxSmallAlloc()*2) ;
IDs = FrNewR(uint32_t,IDs,num_words + new_words) ;
if (!IDs)
{
cout << "*** unable to fill request for "
<< sizeof(uint32_t) * num_words << " bytes! ***" << endl << endl ;
FrMemoryStats() ;
FrNoMemory("while allocating space for word IDs") ;
return 0 ;
}
size_t count = num_words ;
for ( ; list ; list = list->next())
{
for (size_t i = 0 ; i < list->eltsUsed() ; i++)
IDs[count++] = list->getNth(i) ;
}
if (reverse)
{
// since phrases can only efficiently be extended to the left, but we
// need to be able to extend on the right to get a conditional
// probability, reverse the contents of each record
size_t record_start = num_words ;
for (size_t i = num_words ; i < num_words + new_words ; i++)
{
if (IDs[i] >= eor)
{
if (i > record_start + 1) // no need to reverse 1-word records
{
uint32_t *first = &IDs[record_start] ;
uint32_t *last = &IDs[i-1] ;
while (first < last)
{
uint32_t tmp = *first ;
*first++ = *last ;
*last-- = tmp ;
}
}
record_start = i + 1 ;
}
}
}
num_words += new_words ;
return IDs ;
}
static void mem_command(ostream &out, istream &)
{
FrMemoryStats(out) ;
}
static void memv_command(ostream &out, istream &)
{
FrMemoryStats(out,true) ;
return ;
}
