Inversion*
WhitespaceTokenizer_Transform_IMP(WhitespaceTokenizer *self,
Inversion *inversion) {
Inversion *new_inversion = Inversion_new(NULL);
Token *token;
while (NULL != (token = Inversion_Next(inversion))) {
WhitespaceTokenizer_Tokenize_Str(self, Token_Get_Text(token),
Token_Get_Len(token), new_inversion);
}
return new_inversion;
}
ByteBuf*
HLWriter_TV_Buf_IMP(HighlightWriter *self, Inversion *inversion) {
const char *last_text = "";
size_t last_len = 0;
ByteBuf *tv_buf = BB_new(20 + Inversion_Get_Size(inversion) * 8);
uint32_t num_postings = 0;
Token **tokens;
uint32_t freq;
UNUSED_VAR(self);
// Leave space for a c32 indicating the number of postings.
BB_Set_Size(tv_buf, C32_MAX_BYTES);
Inversion_Reset(inversion);
while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
Token *token = *tokens;
char *const token_text = Token_Get_Text(token);
const int32_t token_len = Token_Get_Len(token);
int32_t overlap = StrHelp_overlap(last_text, token_text,
last_len, token_len);
char *ptr;
char *orig;
size_t old_size = BB_Get_Size(tv_buf);
size_t new_size = old_size
+ C32_MAX_BYTES // overlap
+ C32_MAX_BYTES // length of string diff
+ (token_len - overlap) // diff char data
+ C32_MAX_BYTES // num prox
+ (C32_MAX_BYTES * freq * 3); // pos data
// Allocate for worst-case scenario.
ptr = BB_Grow(tv_buf, new_size);
orig = ptr;
ptr += old_size;
// Track number of postings.
num_postings += 1;
// Append the string diff to the tv_buf.
NumUtil_encode_c32(overlap, &ptr);
NumUtil_encode_c32((token_len - overlap), &ptr);
memcpy(ptr, (token_text + overlap), (token_len - overlap));
ptr += token_len - overlap;
// Save text and text_len for comparison next loop.
last_text = token_text;
last_len = token_len;
// Append the number of positions for this term.
NumUtil_encode_c32(freq, &ptr);
do {
// Add position, start_offset, and end_offset to tv_buf.
NumUtil_encode_c32(Token_Get_Pos(token), &ptr);
NumUtil_encode_c32(Token_Get_Start_Offset(token), &ptr);
NumUtil_encode_c32(Token_Get_End_Offset(token), &ptr);
} while (--freq && (token = *++tokens));
// Set new byte length.
BB_Set_Size(tv_buf, ptr - orig);
}
// Go back and start the term vector string with the posting count.
char *dest = BB_Get_Buf(tv_buf);
NumUtil_encode_padded_c32(num_postings, &dest);
return tv_buf;
}
