uint32_t *FrAugmentWordIDArray(const FrWordIDList *list, uint32_t *prevIDs, size_t &num_words, uint32_t eor, bool reverse) { size_t new_words = FrCountWordIDs(list) ; // as we may have trouble requesting a memory block for the full count // right off the bat, work around the issue by first allocating a // smallish block and then resizing it uint32_t *IDs = prevIDs ; if (!IDs && num_words + new_words > 100000) IDs = (uint32_t*)FrMalloc(FrMaxSmallAlloc()*2) ; IDs = FrNewR(uint32_t,IDs,num_words + new_words) ; if (!IDs) { cout << "*** unable to fill request for " << sizeof(uint32_t) * num_words << " bytes! ***" << endl << endl ; FrMemoryStats() ; FrNoMemory("while allocating space for word IDs") ; return 0 ; } size_t count = num_words ; for ( ; list ; list = list->next()) { for (size_t i = 0 ; i < list->eltsUsed() ; i++) IDs[count++] = list->getNth(i) ; } if (reverse) { // since phrases can only efficiently be extended to the left, but we // need to be able to extend on the right to get a conditional // probability, reverse the contents of each record size_t record_start = num_words ; for (size_t i = num_words ; i < num_words + new_words ; i++) { if (IDs[i] >= eor) { if (i > record_start + 1) // no need to reverse 1-word records { uint32_t *first = &IDs[record_start] ; uint32_t *last = &IDs[i-1] ; while (first < last) { uint32_t tmp = *first ; *first++ = *last ; *last-- = tmp ; } } record_start = i + 1 ; } } } num_words += new_words ; return IDs ; }
static void mem_command(ostream &out, istream &) { FrMemoryStats(out) ; }
static void memv_command(ostream &out, istream &) { FrMemoryStats(out,true) ; return ; }