const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
// This method can be called with invalid terminal id during GC.
return ProbabilityEntry(0 /* flags */, NOT_A_PROBABILITY);
}
const BufferWithExtendableBuffer *const buffer = getBuffer();
int entryPos = getEntryPos(terminalId);
const int flags = buffer->readUintAndAdvancePosition(
Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &entryPos);
const int probability = buffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, &entryPos);
if (mHasHistoricalInfo) {
const int timestamp = buffer->readUintAndAdvancePosition(
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &entryPos);
const int level = buffer->readUintAndAdvancePosition(
Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &entryPos);
const int count = buffer->readUintAndAdvancePosition(
Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &entryPos);
// Hack for better migration.
const HistoricalInfo historicalInfo(timestamp, level, count + level);
return ProbabilityEntry(flags, probability, &historicalInfo);
} else {
return ProbabilityEntry(flags, probability);
}
}
bool TerminalPositionLookupTable::setTerminalPtNodePosition(
const int terminalId, const int terminalPtNodePos) {
if (terminalId < 0) {
return NOT_A_DICT_POS;
}
while (terminalId >= mSize) {
// Write new entry.
if (!getWritableBuffer()->writeUint(Ver4DictConstants::NOT_A_TERMINAL_ADDRESS,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(mSize))) {
return false;
}
mSize++;
}
const int terminalPos = (terminalPtNodePos != NOT_A_DICT_POS) ?
terminalPtNodePos : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS;
return getWritableBuffer()->writeUint(terminalPos,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
}
int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
return NOT_A_DICT_POS;
}
const int terminalPos = getBuffer()->readUint(
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
return (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) ?
NOT_A_DICT_POS : terminalPos;
}
bool TerminalPositionLookupTable::runGCTerminalIds(TerminalIdMap *const terminalIdMap) {
int removedEntryCount = 0;
int nextNewTerminalId = 0;
for (int i = 0; i < mSize; ++i) {
const int terminalPos = getBuffer()->readUint(
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(i));
if (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) {
// This entry is a garbage.
removedEntryCount++;
} else {
// Give a new terminal id to the entry.
if (!getWritableBuffer()->writeUint(terminalPos,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
getEntryPos(nextNewTerminalId))) {
return false;
}
// Memorize the mapping to the old terminal id to the new terminal id.
terminalIdMap->insert(TerminalIdMap::value_type(i, nextNewTerminalId));
nextNewTerminalId++;
}
}
mSize = nextNewTerminalId;
return true;
}
bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
for (int i = 0; i < mSize; ++i) {
const ProbabilityEntry probabilityEntry = getProbabilityEntry(i);
if (!probabilityDictContentToWrite.setProbabilityEntry(i, &probabilityEntry)) {
AKLOGE("Cannot set probability entry in flushToFile. terminalId: %d", i);
return false;
}
}
return probabilityDictContentToWrite.flush(dictPath,
Ver4DictConstants::FREQ_FILE_EXTENSION);
} else {
return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
}
}
bool TerminalPositionLookupTable::flushToFile(FILE *const file) const {
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
// table and write the new table to the file.
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
TerminalPositionLookupTable lookupTableToWrite;
for (int i = 0; i < mSize; ++i) {
const int terminalPtNodePosition = getTerminalPtNodePosition(i);
if (!lookupTableToWrite.setTerminalPtNodePosition(i, terminalPtNodePosition)) {
AKLOGE("Cannot set terminal position to lookupTableToWrite."
" terminalId: %d, position: %d", i, terminalPtNodePosition);
return false;
}
}
return lookupTableToWrite.flush(file);
} else {
// We can simply use this lookup table because the buffer size has not been
// changed.
return flush(file);
}
}
bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
const ProbabilityEntry *const probabilityEntry) {
if (terminalId < 0) {
return false;
}
const int entryPos = getEntryPos(terminalId);
if (terminalId >= mSize) {
ProbabilityEntry dummyEntry;
// Write new entry.
int writingPos = getBuffer()->getTailPosition();
while (writingPos <= entryPos) {
// Fulfilling with dummy entries until writingPos.
if (!writeEntry(&dummyEntry, writingPos)) {
AKLOGE("Cannot write dummy entry. pos: %d, mSize: %d", writingPos, mSize);
return false;
}
writingPos += getEntrySize();
}
mSize = terminalId + 1;
}
return writeEntry(probabilityEntry, entryPos);
}