static void
S_add_last_term_to_ix(LexiconWriter *self, char *last_text, size_t last_size)
{
OutStream *const ix_out = self->ix_out;
OutStream *const ixix_out = self->ixix_out;
TermInfo *const last_tinfo = self->last_tinfo;
/* Write file pointer to index record. */
OutStream_Write_U64(ixix_out, OutStream_Tell(ix_out));
/* Write term text. */
OutStream_Write_C32(ix_out, last_size);
OutStream_Write_Bytes(ix_out, last_text, last_size);
/* Write doc_freq. */
OutStream_Write_C32(ix_out, last_tinfo->doc_freq);
/* Write postings file pointer. */
OutStream_Write_C64(ix_out, last_tinfo->post_filepos);
/* Write skip file pointer (maybe). */
if (last_tinfo->doc_freq >= self->skip_interval) {
OutStream_Write_C64(ix_out, last_tinfo->skip_filepos);
}
/* Write file pointer to main record. */
OutStream_Write_C64(ix_out, OutStream_Tell(self->dat_out));
/* Keep track of how many terms have been added to lexicon.ix. */
self->ix_count++;
}
void
Hash_serialize(Hash *self, OutStream *outstream) {
Obj *key;
Obj *val;
uint32_t charbuf_count = 0;
OutStream_Write_C32(outstream, self->size);
// Write CharBuf keys first. CharBuf keys are the common case; grouping
// them together is a form of run-length-encoding and saves space, since
// we omit the per-key class name.
Hash_Iterate(self);
while (Hash_Next(self, &key, &val)) {
if (Obj_Is_A(key, CHARBUF)) {
charbuf_count++;
}
}
OutStream_Write_C32(outstream, charbuf_count);
Hash_Iterate(self);
while (Hash_Next(self, &key, &val)) {
if (Obj_Is_A(key, CHARBUF)) {
Obj_Serialize(key, outstream);
FREEZE(val, outstream);
}
}
// Punt on the classes of the remaining keys.
Hash_Iterate(self);
while (Hash_Next(self, &key, &val)) {
if (!Obj_Is_A(key, CHARBUF)) {
FREEZE(key, outstream);
FREEZE(val, outstream);
}
}
}
void
Freezer_serialize_varray(Vector *array, OutStream *outstream) {
uint32_t last_valid_tick = 0;
uint32_t size = (uint32_t)Vec_Get_Size(array);
OutStream_Write_C32(outstream, size);
for (uint32_t i = 0; i < size; i++) {
Obj *elem = Vec_Fetch(array, i);
if (elem) {
OutStream_Write_C32(outstream, i - last_valid_tick);
FREEZE(elem, outstream);
last_valid_tick = i;
}
}
// Terminate.
OutStream_Write_C32(outstream, size - last_valid_tick);
}
void
Doc_Serialize_IMP(Doc *self, OutStream *outstream) {
DocIVARS *const ivars = Doc_IVARS(self);
Hash *hash = (Hash*)ivars->fields;
Freezer_serialize_hash(hash, outstream);
OutStream_Write_C32(outstream, ivars->doc_id);
}
void
TV_Serialize_IMP(TermVector *self, OutStream *target) {
TermVectorIVARS *const ivars = TV_IVARS(self);
int32_t *posits = I32Arr_IVARS(ivars->positions)->ints;
int32_t *starts = I32Arr_IVARS(ivars->start_offsets)->ints;
int32_t *ends = I32Arr_IVARS(ivars->start_offsets)->ints;
Freezer_serialize_string(ivars->field, target);
Freezer_serialize_string(ivars->text, target);
OutStream_Write_C64(target, ivars->num_pos);
for (size_t i = 0; i < ivars->num_pos; i++) {
OutStream_Write_C32(target, posits[i]);
OutStream_Write_C32(target, starts[i]);
OutStream_Write_C32(target, ends[i]);
}
}
void
RawPost_write_record(RawPosting *self, OutStream *outstream,
i32_t last_doc_id)
{
const u32_t delta_doc = self->doc_id - last_doc_id;
char *const aux_content = self->blob + self->content_len;
if (self->freq == 1) {
const u32_t doc_code = (delta_doc << 1) | 1;
OutStream_Write_C32(outstream, doc_code);
}
else {
const u32_t doc_code = delta_doc << 1;
OutStream_Write_C32(outstream, doc_code);
OutStream_Write_C32(outstream, self->freq);
}
OutStream_Write_Bytes(outstream, aux_content, self->aux_len);
}
void
VA_serialize(VArray *self, OutStream *outstream)
{
u32_t i;
u32_t last_valid_tick = 0;
OutStream_Write_C32(outstream, self->size);
for (i = 0; i < self->size; i++) {
Obj *elem = self->elems[i];
if (elem) {
OutStream_Write_C32(outstream, i - last_valid_tick);
FREEZE(elem, outstream);
last_valid_tick = i;
}
}
/* Terminate. */
OutStream_Write_C32(outstream, self->size - last_valid_tick);
}
void
ProximityQuery_Serialize_IMP(ProximityQuery *self, OutStream *outstream) {
ProximityQueryIVARS *const ivars = ProximityQuery_IVARS(self);
OutStream_Write_F32(outstream, ivars->boost);
Freezer_serialize_string(ivars->field, outstream);
Freezer_serialize_varray(ivars->terms, outstream);
OutStream_Write_C32(outstream, ivars->within);
}
void
MatchDoc_serialize(MatchDoc *self, OutStream *outstream)
{
OutStream_Write_C32(outstream, self->doc_id);
OutStream_Write_Float(outstream, self->score);
OutStream_Write_U8(outstream, self->values ? 1 : 0);
if (self->values) { VA_Serialize(self->values, outstream); }
}
void
MatchPostWriter_write_posting(MatchPostingWriter *self, RawPosting *posting) {
OutStream *const outstream = self->outstream;
const int32_t doc_id = posting->doc_id;
const uint32_t delta_doc = doc_id - self->last_doc_id;
char *const aux_content = posting->blob + posting->content_len;
if (posting->freq == 1) {
const uint32_t doc_code = (delta_doc << 1) | 1;
OutStream_Write_C32(outstream, doc_code);
}
else {
const uint32_t doc_code = delta_doc << 1;
OutStream_Write_C32(outstream, doc_code);
OutStream_Write_C32(outstream, posting->freq);
}
OutStream_Write_Bytes(outstream, aux_content, posting->aux_len);
self->last_doc_id = doc_id;
}
void
TV_serialize(TermVector *self, OutStream *target)
{
u32_t i;
i32_t *posits = self->positions->ints;
i32_t *starts = self->start_offsets->ints;
i32_t *ends = self->start_offsets->ints;
CB_Serialize(self->field, target);
CB_Serialize(self->text, target);
OutStream_Write_C32(target, self->num_pos);
for (i = 0; i < self->num_pos; i++) {
OutStream_Write_C32(target, posits[i]);
OutStream_Write_C32(target, starts[i]);
OutStream_Write_C32(target, ends[i]);
}
}
void
SortSpec_serialize(SortSpec *self, OutStream *target) {
SortSpecIVARS *const ivars = SortSpec_IVARS(self);
uint32_t num_rules = VA_Get_Size(ivars->rules);
OutStream_Write_C32(target, num_rules);
for (uint32_t i = 0; i < num_rules; i++) {
SortRule *rule = (SortRule*)VA_Fetch(ivars->rules, i);
SortRule_Serialize(rule, target);
}
}
void
MatchPostWriter_Write_Posting_IMP(MatchPostingWriter *self, RawPosting *posting) {
MatchPostingWriterIVARS *const ivars = MatchPostWriter_IVARS(self);
RawPostingIVARS *const posting_ivars = RawPost_IVARS(posting);
OutStream *const outstream = ivars->outstream;
const int32_t doc_id = posting_ivars->doc_id;
const uint32_t delta_doc = doc_id - ivars->last_doc_id;
char *const aux_content = posting_ivars->blob
+ posting_ivars->content_len;
if (posting_ivars->freq == 1) {
const uint32_t doc_code = (delta_doc << 1) | 1;
OutStream_Write_C32(outstream, doc_code);
}
else {
const uint32_t doc_code = delta_doc << 1;
OutStream_Write_C32(outstream, doc_code);
OutStream_Write_C32(outstream, posting_ivars->freq);
}
OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len);
ivars->last_doc_id = doc_id;
}
void
ProximityCompiler_Serialize_IMP(ProximityCompiler *self,
OutStream *outstream) {
ProximityCompiler_Serialize_t super_serialize
= SUPER_METHOD_PTR(PROXIMITYCOMPILER, LUCY_ProximityCompiler_Serialize);
super_serialize(self, outstream);
ProximityCompilerIVARS *const ivars = ProximityCompiler_IVARS(self);
OutStream_Write_F32(outstream, ivars->idf);
OutStream_Write_F32(outstream, ivars->raw_weight);
OutStream_Write_F32(outstream, ivars->query_norm_factor);
OutStream_Write_F32(outstream, ivars->normalized_weight);
OutStream_Write_C32(outstream, ivars->within);
}
void
TextTermStepper_Write_Key_Frame_IMP(TextTermStepper *self,
OutStream *outstream, Obj *value) {
TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self);
CharBuf *charbuf = (CharBuf*)ivars->value;
CB_Mimic(charbuf, value);
const char *buf = CB_Get_Ptr8(charbuf);
size_t size = CB_Get_Size(charbuf);
OutStream_Write_C32(outstream, size);
OutStream_Write_Bytes(outstream, buf, size);
// Invalidate string.
DECREF(ivars->string);
ivars->string = NULL;
}
void
Freezer_serialize_hash(Hash *hash, OutStream *outstream) {
uint32_t hash_size = Hash_Get_Size(hash);
OutStream_Write_C32(outstream, hash_size);
HashIterator *iter = HashIter_new(hash);
while (HashIter_Next(iter)) {
String *key = HashIter_Get_Key(iter);
Obj *val = HashIter_Get_Value(iter);
Freezer_serialize_string(key, outstream);
FREEZE(val, outstream);
}
DECREF(iter);
}
void
HLWriter_Add_Inverted_Doc_IMP(HighlightWriter *self, Inverter *inverter,
int32_t doc_id) {
HighlightWriterIVARS *const ivars = HLWriter_IVARS(self);
OutStream *dat_out = S_lazy_init(self);
OutStream *ix_out = ivars->ix_out;
int64_t filepos = OutStream_Tell(dat_out);
uint32_t num_highlightable = 0;
int32_t expected = (int32_t)(OutStream_Tell(ix_out) / 8);
// Verify doc id.
if (doc_id != expected) {
THROW(ERR, "Expected doc id %i32 but got %i32", expected, doc_id);
}
// Write index data.
OutStream_Write_I64(ix_out, filepos);
// Count, then write number of highlightable fields.
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Is_A(type, FULLTEXTTYPE)
&& FullTextType_Highlightable((FullTextType*)type)
) {
num_highlightable++;
}
}
OutStream_Write_C32(dat_out, num_highlightable);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Is_A(type, FULLTEXTTYPE)
&& FullTextType_Highlightable((FullTextType*)type)
) {
String *field = Inverter_Get_Field_Name(inverter);
Inversion *inversion = Inverter_Get_Inversion(inverter);
ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion);
Freezer_serialize_string(field, dat_out);
Freezer_serialize_bytebuf(tv_buf, dat_out);
DECREF(tv_buf);
}
}
}
void
HLWriter_add_inverted_doc(HighlightWriter *self, Inverter *inverter,
i32_t doc_id)
{
OutStream *dat_out = S_lazy_init(self);
OutStream *ix_out = self->ix_out;
i64_t filepos = OutStream_Tell(dat_out);
u32_t num_highlightable = 0;
i32_t expected = (i32_t)(OutStream_Tell(ix_out) / 8);
/* Verify doc id. */
if (doc_id != expected)
THROW("Expected doc id %i32 but got %i32", expected, doc_id);
/* Write index data. */
OutStream_Write_U64(ix_out, filepos);
/* Count, then write number of highlightable fields. */
Inverter_Iter_Init(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if ( OBJ_IS_A(type, FULLTEXTTYPE)
&& FullTextType_Highlightable(type)
) {
num_highlightable++;
}
}
OutStream_Write_C32(dat_out, num_highlightable);
Inverter_Iter_Init(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if ( OBJ_IS_A(type, FULLTEXTTYPE)
&& FullTextType_Highlightable(type)
) {
CharBuf *field = Inverter_Get_Field_Name(inverter);
Inversion *inversion = Inverter_Get_Inversion(inverter);
ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion);
CB_Serialize(field, dat_out);
BB_Serialize(tv_buf, dat_out);
DECREF(tv_buf);
}
}
}
void
MatchTInfoStepper_write_key_frame(MatchTermInfoStepper *self,
OutStream *outstream, Obj *value) {
TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO);
int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo);
// Write doc_freq.
OutStream_Write_C32(outstream, doc_freq);
// Write postings file pointer.
OutStream_Write_C64(outstream, tinfo->post_filepos);
// Write skip file pointer (maybe).
if (doc_freq >= self->skip_interval) {
OutStream_Write_C64(outstream, tinfo->skip_filepos);
}
TInfo_Mimic((TermInfo*)self->value, (Obj*)tinfo);
}
void
TextTermStepper_Write_Delta_IMP(TextTermStepper *self, OutStream *outstream,
Obj *value) {
TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self);
CharBuf *charbuf = (CharBuf*)ivars->value;
const char *last_text = CB_Get_Ptr8(charbuf);
size_t last_size = CB_Get_Size(charbuf);
const char *new_text = NULL;
size_t new_size = 0;
if (Obj_is_a(value, STRING)) {
String *new_string = (String*)value;
new_text = Str_Get_Ptr8(new_string);
new_size = Str_Get_Size(new_string);
}
else if (Obj_is_a(value, CHARBUF)) {
CharBuf *new_charbuf = (CharBuf*)value;
new_text = CB_Get_Ptr8(new_charbuf);
new_size = CB_Get_Size(new_charbuf);
}
else {
THROW(ERR, "'value' must be a String or CharBuf");
}
// Count how many bytes the strings share at the top.
const int32_t overlap = StrHelp_overlap(last_text, new_text,
last_size, new_size);
const char *const diff_start_str = new_text + overlap;
const size_t diff_len = new_size - overlap;
// Write number of common bytes and common bytes.
OutStream_Write_C32(outstream, overlap);
OutStream_Write_String(outstream, diff_start_str, diff_len);
// Update value.
CB_Mimic_Utf8(charbuf, new_text, new_size);
// Invalidate string.
DECREF(ivars->string);
ivars->string = NULL;
}
void
MatchTInfoStepper_Write_Key_Frame_IMP(MatchTermInfoStepper *self,
OutStream *outstream, Obj *value) {
MatchTermInfoStepperIVARS *const ivars = MatchTInfoStepper_IVARS(self);
TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO);
int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo);
TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)value);
// Write doc_freq.
OutStream_Write_C32(outstream, doc_freq);
// Write postings file pointer.
OutStream_Write_C64(outstream, tinfo_ivars->post_filepos);
// Write skip file pointer (maybe).
if (doc_freq >= ivars->skip_interval) {
OutStream_Write_C64(outstream, tinfo_ivars->skip_filepos);
}
TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo);
}
void
MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self,
OutStream *outstream, Obj *value) {
MatchTermInfoStepperIVARS *const ivars = MatchTInfoStepper_IVARS(self);
TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO);
TermInfo *last_tinfo = (TermInfo*)ivars->value;
int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo);
int64_t post_delta = TInfo_IVARS(tinfo)->post_filepos
- TInfo_IVARS(last_tinfo)->post_filepos;
// Write doc_freq.
OutStream_Write_C32(outstream, doc_freq);
// Write postings file pointer delta.
OutStream_Write_C64(outstream, post_delta);
// Write skip file pointer (maybe).
if (doc_freq >= ivars->skip_interval) {
OutStream_Write_C64(outstream, TInfo_IVARS(tinfo)->skip_filepos);
}
TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo);
}
void
DocWriter_add_inverted_doc(DocWriter *self, Inverter *inverter,
int32_t doc_id) {
OutStream *dat_out = S_lazy_init(self);
OutStream *ix_out = self->ix_out;
uint32_t num_stored = 0;
int64_t start = OutStream_Tell(dat_out);
int64_t expected = OutStream_Tell(ix_out) / 8;
// Verify doc id.
if (doc_id != expected) {
THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id);
}
// Write the number of stored fields.
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) { num_stored++; }
}
OutStream_Write_C32(dat_out, num_stored);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
// Only store fields marked as "stored".
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) {
CharBuf *field = Inverter_Get_Field_Name(inverter);
Obj *value = Inverter_Get_Value(inverter);
CB_Serialize(field, dat_out);
Obj_Serialize(value, dat_out);
}
}
// Write file pointer.
OutStream_Write_I64(ix_out, start);
}
void
TextTermStepper_write_delta(TextTermStepper *self, OutStream *outstream,
Obj *value)
{
CharBuf *new_value = (CharBuf*)CERTIFY(value, CHARBUF);
CharBuf *last_value = (CharBuf*)self->value;
char *new_text = (char*)CB_Get_Ptr8(new_value);
size_t new_size = CB_Get_Size(new_value);
char *last_text = (char*)CB_Get_Ptr8(last_value);
size_t last_size = CB_Get_Size(last_value);
// Count how many bytes the strings share at the top.
const int32_t overlap = StrHelp_overlap(last_text, new_text,
last_size, new_size);
const char *const diff_start_str = new_text + overlap;
const size_t diff_len = new_size - overlap;
// Write number of common bytes and common bytes.
OutStream_Write_C32(outstream, overlap);
OutStream_Write_String(outstream, diff_start_str, diff_len);
// Update value.
CB_Mimic((CharBuf*)self->value, value);
}
void
TopDocs_Serialize_IMP(TopDocs *self, OutStream *outstream) {
TopDocsIVARS *const ivars = TopDocs_IVARS(self);
Freezer_serialize_varray(ivars->match_docs, outstream);
OutStream_Write_C32(outstream, ivars->total_hits);
}
void
Freezer_serialize_blob(Blob *blob, OutStream *outstream) {
size_t size = Blob_Get_Size(blob);
OutStream_Write_C32(outstream, size);
OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size);
}
void
DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
int32_t doc_id) {
DocWriterIVARS *const ivars = DocWriter_IVARS(self);
OutStream *dat_out = S_lazy_init(self);
OutStream *ix_out = ivars->ix_out;
uint32_t num_stored = 0;
int64_t start = OutStream_Tell(dat_out);
int64_t expected = OutStream_Tell(ix_out) / 8;
// Verify doc id.
if (doc_id != expected) {
THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id);
}
// Write the number of stored fields.
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) { num_stored++; }
}
OutStream_Write_C32(dat_out, num_stored);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
// Only store fields marked as "stored".
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) {
String *field = Inverter_Get_Field_Name(inverter);
Obj *value = Inverter_Get_Value(inverter);
Freezer_serialize_string(field, dat_out);
switch (FType_Primitive_ID(type) & FType_PRIMITIVE_ID_MASK) {
case FType_TEXT: {
const char *buf = Str_Get_Ptr8((String*)value);
size_t size = Str_Get_Size((String*)value);
OutStream_Write_C32(dat_out, size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
case FType_BLOB: {
char *buf = BB_Get_Buf((ByteBuf*)value);
size_t size = BB_Get_Size((ByteBuf*)value);
OutStream_Write_C32(dat_out, size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
case FType_INT32: {
int32_t val = Int32_Get_Value((Integer32*)value);
OutStream_Write_C32(dat_out, val);
break;
}
case FType_INT64: {
int64_t val = Int64_Get_Value((Integer64*)value);
OutStream_Write_C64(dat_out, val);
break;
}
case FType_FLOAT32: {
float val = Float32_Get_Value((Float32*)value);
OutStream_Write_F32(dat_out, val);
break;
}
case FType_FLOAT64: {
double val = Float64_Get_Value((Float64*)value);
OutStream_Write_F64(dat_out, val);
break;
}
default:
THROW(ERR, "Unrecognized type: %o", type);
}
}
}
// Write file pointer.
OutStream_Write_I64(ix_out, start);
}
void
TopDocs_serialize(TopDocs *self, OutStream *outstream)
{
VA_Serialize(self->match_docs, outstream);
OutStream_Write_C32(outstream, self->total_hits);
}