static void S_add_last_term_to_ix(LexiconWriter *self, char *last_text, size_t last_size) { OutStream *const ix_out = self->ix_out; OutStream *const ixix_out = self->ixix_out; TermInfo *const last_tinfo = self->last_tinfo; /* Write file pointer to index record. */ OutStream_Write_U64(ixix_out, OutStream_Tell(ix_out)); /* Write term text. */ OutStream_Write_C32(ix_out, last_size); OutStream_Write_Bytes(ix_out, last_text, last_size); /* Write doc_freq. */ OutStream_Write_C32(ix_out, last_tinfo->doc_freq); /* Write postings file pointer. */ OutStream_Write_C64(ix_out, last_tinfo->post_filepos); /* Write skip file pointer (maybe). */ if (last_tinfo->doc_freq >= self->skip_interval) { OutStream_Write_C64(ix_out, last_tinfo->skip_filepos); } /* Write file pointer to main record. */ OutStream_Write_C64(ix_out, OutStream_Tell(self->dat_out)); /* Keep track of how many terms have been added to lexicon.ix. */ self->ix_count++; }
void Hash_serialize(Hash *self, OutStream *outstream) { Obj *key; Obj *val; uint32_t charbuf_count = 0; OutStream_Write_C32(outstream, self->size); // Write CharBuf keys first. CharBuf keys are the common case; grouping // them together is a form of run-length-encoding and saves space, since // we omit the per-key class name. Hash_Iterate(self); while (Hash_Next(self, &key, &val)) { if (Obj_Is_A(key, CHARBUF)) { charbuf_count++; } } OutStream_Write_C32(outstream, charbuf_count); Hash_Iterate(self); while (Hash_Next(self, &key, &val)) { if (Obj_Is_A(key, CHARBUF)) { Obj_Serialize(key, outstream); FREEZE(val, outstream); } } // Punt on the classes of the remaining keys. Hash_Iterate(self); while (Hash_Next(self, &key, &val)) { if (!Obj_Is_A(key, CHARBUF)) { FREEZE(key, outstream); FREEZE(val, outstream); } } }
void Freezer_serialize_varray(Vector *array, OutStream *outstream) { uint32_t last_valid_tick = 0; uint32_t size = (uint32_t)Vec_Get_Size(array); OutStream_Write_C32(outstream, size); for (uint32_t i = 0; i < size; i++) { Obj *elem = Vec_Fetch(array, i); if (elem) { OutStream_Write_C32(outstream, i - last_valid_tick); FREEZE(elem, outstream); last_valid_tick = i; } } // Terminate. OutStream_Write_C32(outstream, size - last_valid_tick); }
void Doc_Serialize_IMP(Doc *self, OutStream *outstream) { DocIVARS *const ivars = Doc_IVARS(self); Hash *hash = (Hash*)ivars->fields; Freezer_serialize_hash(hash, outstream); OutStream_Write_C32(outstream, ivars->doc_id); }
void TV_Serialize_IMP(TermVector *self, OutStream *target) { TermVectorIVARS *const ivars = TV_IVARS(self); int32_t *posits = I32Arr_IVARS(ivars->positions)->ints; int32_t *starts = I32Arr_IVARS(ivars->start_offsets)->ints; int32_t *ends = I32Arr_IVARS(ivars->start_offsets)->ints; Freezer_serialize_string(ivars->field, target); Freezer_serialize_string(ivars->text, target); OutStream_Write_C64(target, ivars->num_pos); for (size_t i = 0; i < ivars->num_pos; i++) { OutStream_Write_C32(target, posits[i]); OutStream_Write_C32(target, starts[i]); OutStream_Write_C32(target, ends[i]); } }
void RawPost_write_record(RawPosting *self, OutStream *outstream, i32_t last_doc_id) { const u32_t delta_doc = self->doc_id - last_doc_id; char *const aux_content = self->blob + self->content_len; if (self->freq == 1) { const u32_t doc_code = (delta_doc << 1) | 1; OutStream_Write_C32(outstream, doc_code); } else { const u32_t doc_code = delta_doc << 1; OutStream_Write_C32(outstream, doc_code); OutStream_Write_C32(outstream, self->freq); } OutStream_Write_Bytes(outstream, aux_content, self->aux_len); }
void VA_serialize(VArray *self, OutStream *outstream) { u32_t i; u32_t last_valid_tick = 0; OutStream_Write_C32(outstream, self->size); for (i = 0; i < self->size; i++) { Obj *elem = self->elems[i]; if (elem) { OutStream_Write_C32(outstream, i - last_valid_tick); FREEZE(elem, outstream); last_valid_tick = i; } } /* Terminate. */ OutStream_Write_C32(outstream, self->size - last_valid_tick); }
void ProximityQuery_Serialize_IMP(ProximityQuery *self, OutStream *outstream) { ProximityQueryIVARS *const ivars = ProximityQuery_IVARS(self); OutStream_Write_F32(outstream, ivars->boost); Freezer_serialize_string(ivars->field, outstream); Freezer_serialize_varray(ivars->terms, outstream); OutStream_Write_C32(outstream, ivars->within); }
void MatchDoc_serialize(MatchDoc *self, OutStream *outstream) { OutStream_Write_C32(outstream, self->doc_id); OutStream_Write_Float(outstream, self->score); OutStream_Write_U8(outstream, self->values ? 1 : 0); if (self->values) { VA_Serialize(self->values, outstream); } }
void MatchPostWriter_write_posting(MatchPostingWriter *self, RawPosting *posting) { OutStream *const outstream = self->outstream; const int32_t doc_id = posting->doc_id; const uint32_t delta_doc = doc_id - self->last_doc_id; char *const aux_content = posting->blob + posting->content_len; if (posting->freq == 1) { const uint32_t doc_code = (delta_doc << 1) | 1; OutStream_Write_C32(outstream, doc_code); } else { const uint32_t doc_code = delta_doc << 1; OutStream_Write_C32(outstream, doc_code); OutStream_Write_C32(outstream, posting->freq); } OutStream_Write_Bytes(outstream, aux_content, posting->aux_len); self->last_doc_id = doc_id; }
void TV_serialize(TermVector *self, OutStream *target) { u32_t i; i32_t *posits = self->positions->ints; i32_t *starts = self->start_offsets->ints; i32_t *ends = self->start_offsets->ints; CB_Serialize(self->field, target); CB_Serialize(self->text, target); OutStream_Write_C32(target, self->num_pos); for (i = 0; i < self->num_pos; i++) { OutStream_Write_C32(target, posits[i]); OutStream_Write_C32(target, starts[i]); OutStream_Write_C32(target, ends[i]); } }
void SortSpec_serialize(SortSpec *self, OutStream *target) { SortSpecIVARS *const ivars = SortSpec_IVARS(self); uint32_t num_rules = VA_Get_Size(ivars->rules); OutStream_Write_C32(target, num_rules); for (uint32_t i = 0; i < num_rules; i++) { SortRule *rule = (SortRule*)VA_Fetch(ivars->rules, i); SortRule_Serialize(rule, target); } }
void MatchPostWriter_Write_Posting_IMP(MatchPostingWriter *self, RawPosting *posting) { MatchPostingWriterIVARS *const ivars = MatchPostWriter_IVARS(self); RawPostingIVARS *const posting_ivars = RawPost_IVARS(posting); OutStream *const outstream = ivars->outstream; const int32_t doc_id = posting_ivars->doc_id; const uint32_t delta_doc = doc_id - ivars->last_doc_id; char *const aux_content = posting_ivars->blob + posting_ivars->content_len; if (posting_ivars->freq == 1) { const uint32_t doc_code = (delta_doc << 1) | 1; OutStream_Write_C32(outstream, doc_code); } else { const uint32_t doc_code = delta_doc << 1; OutStream_Write_C32(outstream, doc_code); OutStream_Write_C32(outstream, posting_ivars->freq); } OutStream_Write_Bytes(outstream, aux_content, posting_ivars->aux_len); ivars->last_doc_id = doc_id; }
void ProximityCompiler_Serialize_IMP(ProximityCompiler *self, OutStream *outstream) { ProximityCompiler_Serialize_t super_serialize = SUPER_METHOD_PTR(PROXIMITYCOMPILER, LUCY_ProximityCompiler_Serialize); super_serialize(self, outstream); ProximityCompilerIVARS *const ivars = ProximityCompiler_IVARS(self); OutStream_Write_F32(outstream, ivars->idf); OutStream_Write_F32(outstream, ivars->raw_weight); OutStream_Write_F32(outstream, ivars->query_norm_factor); OutStream_Write_F32(outstream, ivars->normalized_weight); OutStream_Write_C32(outstream, ivars->within); }
void TextTermStepper_Write_Key_Frame_IMP(TextTermStepper *self, OutStream *outstream, Obj *value) { TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self); CharBuf *charbuf = (CharBuf*)ivars->value; CB_Mimic(charbuf, value); const char *buf = CB_Get_Ptr8(charbuf); size_t size = CB_Get_Size(charbuf); OutStream_Write_C32(outstream, size); OutStream_Write_Bytes(outstream, buf, size); // Invalidate string. DECREF(ivars->string); ivars->string = NULL; }
void Freezer_serialize_hash(Hash *hash, OutStream *outstream) { uint32_t hash_size = Hash_Get_Size(hash); OutStream_Write_C32(outstream, hash_size); HashIterator *iter = HashIter_new(hash); while (HashIter_Next(iter)) { String *key = HashIter_Get_Key(iter); Obj *val = HashIter_Get_Value(iter); Freezer_serialize_string(key, outstream); FREEZE(val, outstream); } DECREF(iter); }
void HLWriter_Add_Inverted_Doc_IMP(HighlightWriter *self, Inverter *inverter, int32_t doc_id) { HighlightWriterIVARS *const ivars = HLWriter_IVARS(self); OutStream *dat_out = S_lazy_init(self); OutStream *ix_out = ivars->ix_out; int64_t filepos = OutStream_Tell(dat_out); uint32_t num_highlightable = 0; int32_t expected = (int32_t)(OutStream_Tell(ix_out) / 8); // Verify doc id. if (doc_id != expected) { THROW(ERR, "Expected doc id %i32 but got %i32", expected, doc_id); } // Write index data. OutStream_Write_I64(ix_out, filepos); // Count, then write number of highlightable fields. Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { FieldType *type = Inverter_Get_Type(inverter); if (FType_Is_A(type, FULLTEXTTYPE) && FullTextType_Highlightable((FullTextType*)type) ) { num_highlightable++; } } OutStream_Write_C32(dat_out, num_highlightable); Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { FieldType *type = Inverter_Get_Type(inverter); if (FType_Is_A(type, FULLTEXTTYPE) && FullTextType_Highlightable((FullTextType*)type) ) { String *field = Inverter_Get_Field_Name(inverter); Inversion *inversion = Inverter_Get_Inversion(inverter); ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion); Freezer_serialize_string(field, dat_out); Freezer_serialize_bytebuf(tv_buf, dat_out); DECREF(tv_buf); } } }
void HLWriter_add_inverted_doc(HighlightWriter *self, Inverter *inverter, i32_t doc_id) { OutStream *dat_out = S_lazy_init(self); OutStream *ix_out = self->ix_out; i64_t filepos = OutStream_Tell(dat_out); u32_t num_highlightable = 0; i32_t expected = (i32_t)(OutStream_Tell(ix_out) / 8); /* Verify doc id. */ if (doc_id != expected) THROW("Expected doc id %i32 but got %i32", expected, doc_id); /* Write index data. */ OutStream_Write_U64(ix_out, filepos); /* Count, then write number of highlightable fields. */ Inverter_Iter_Init(inverter); while (Inverter_Next(inverter)) { FieldType *type = Inverter_Get_Type(inverter); if ( OBJ_IS_A(type, FULLTEXTTYPE) && FullTextType_Highlightable(type) ) { num_highlightable++; } } OutStream_Write_C32(dat_out, num_highlightable); Inverter_Iter_Init(inverter); while (Inverter_Next(inverter)) { FieldType *type = Inverter_Get_Type(inverter); if ( OBJ_IS_A(type, FULLTEXTTYPE) && FullTextType_Highlightable(type) ) { CharBuf *field = Inverter_Get_Field_Name(inverter); Inversion *inversion = Inverter_Get_Inversion(inverter); ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion); CB_Serialize(field, dat_out); BB_Serialize(tv_buf, dat_out); DECREF(tv_buf); } } }
void MatchTInfoStepper_write_key_frame(MatchTermInfoStepper *self, OutStream *outstream, Obj *value) { TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO); int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo); // Write doc_freq. OutStream_Write_C32(outstream, doc_freq); // Write postings file pointer. OutStream_Write_C64(outstream, tinfo->post_filepos); // Write skip file pointer (maybe). if (doc_freq >= self->skip_interval) { OutStream_Write_C64(outstream, tinfo->skip_filepos); } TInfo_Mimic((TermInfo*)self->value, (Obj*)tinfo); }
void TextTermStepper_Write_Delta_IMP(TextTermStepper *self, OutStream *outstream, Obj *value) { TextTermStepperIVARS *const ivars = TextTermStepper_IVARS(self); CharBuf *charbuf = (CharBuf*)ivars->value; const char *last_text = CB_Get_Ptr8(charbuf); size_t last_size = CB_Get_Size(charbuf); const char *new_text = NULL; size_t new_size = 0; if (Obj_is_a(value, STRING)) { String *new_string = (String*)value; new_text = Str_Get_Ptr8(new_string); new_size = Str_Get_Size(new_string); } else if (Obj_is_a(value, CHARBUF)) { CharBuf *new_charbuf = (CharBuf*)value; new_text = CB_Get_Ptr8(new_charbuf); new_size = CB_Get_Size(new_charbuf); } else { THROW(ERR, "'value' must be a String or CharBuf"); } // Count how many bytes the strings share at the top. const int32_t overlap = StrHelp_overlap(last_text, new_text, last_size, new_size); const char *const diff_start_str = new_text + overlap; const size_t diff_len = new_size - overlap; // Write number of common bytes and common bytes. OutStream_Write_C32(outstream, overlap); OutStream_Write_String(outstream, diff_start_str, diff_len); // Update value. CB_Mimic_Utf8(charbuf, new_text, new_size); // Invalidate string. DECREF(ivars->string); ivars->string = NULL; }
void MatchTInfoStepper_Write_Key_Frame_IMP(MatchTermInfoStepper *self, OutStream *outstream, Obj *value) { MatchTermInfoStepperIVARS *const ivars = MatchTInfoStepper_IVARS(self); TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO); int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo); TermInfoIVARS *const tinfo_ivars = TInfo_IVARS((TermInfo*)value); // Write doc_freq. OutStream_Write_C32(outstream, doc_freq); // Write postings file pointer. OutStream_Write_C64(outstream, tinfo_ivars->post_filepos); // Write skip file pointer (maybe). if (doc_freq >= ivars->skip_interval) { OutStream_Write_C64(outstream, tinfo_ivars->skip_filepos); } TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo); }
void MatchTInfoStepper_Write_Delta_IMP(MatchTermInfoStepper *self, OutStream *outstream, Obj *value) { MatchTermInfoStepperIVARS *const ivars = MatchTInfoStepper_IVARS(self); TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO); TermInfo *last_tinfo = (TermInfo*)ivars->value; int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo); int64_t post_delta = TInfo_IVARS(tinfo)->post_filepos - TInfo_IVARS(last_tinfo)->post_filepos; // Write doc_freq. OutStream_Write_C32(outstream, doc_freq); // Write postings file pointer delta. OutStream_Write_C64(outstream, post_delta); // Write skip file pointer (maybe). if (doc_freq >= ivars->skip_interval) { OutStream_Write_C64(outstream, TInfo_IVARS(tinfo)->skip_filepos); } TInfo_Mimic((TermInfo*)ivars->value, (Obj*)tinfo); }
void DocWriter_add_inverted_doc(DocWriter *self, Inverter *inverter, int32_t doc_id) { OutStream *dat_out = S_lazy_init(self); OutStream *ix_out = self->ix_out; uint32_t num_stored = 0; int64_t start = OutStream_Tell(dat_out); int64_t expected = OutStream_Tell(ix_out) / 8; // Verify doc id. if (doc_id != expected) { THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id); } // Write the number of stored fields. Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { FieldType *type = Inverter_Get_Type(inverter); if (FType_Stored(type)) { num_stored++; } } OutStream_Write_C32(dat_out, num_stored); Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { // Only store fields marked as "stored". FieldType *type = Inverter_Get_Type(inverter); if (FType_Stored(type)) { CharBuf *field = Inverter_Get_Field_Name(inverter); Obj *value = Inverter_Get_Value(inverter); CB_Serialize(field, dat_out); Obj_Serialize(value, dat_out); } } // Write file pointer. OutStream_Write_I64(ix_out, start); }
void TextTermStepper_write_delta(TextTermStepper *self, OutStream *outstream, Obj *value) { CharBuf *new_value = (CharBuf*)CERTIFY(value, CHARBUF); CharBuf *last_value = (CharBuf*)self->value; char *new_text = (char*)CB_Get_Ptr8(new_value); size_t new_size = CB_Get_Size(new_value); char *last_text = (char*)CB_Get_Ptr8(last_value); size_t last_size = CB_Get_Size(last_value); // Count how many bytes the strings share at the top. const int32_t overlap = StrHelp_overlap(last_text, new_text, last_size, new_size); const char *const diff_start_str = new_text + overlap; const size_t diff_len = new_size - overlap; // Write number of common bytes and common bytes. OutStream_Write_C32(outstream, overlap); OutStream_Write_String(outstream, diff_start_str, diff_len); // Update value. CB_Mimic((CharBuf*)self->value, value); }
void TopDocs_Serialize_IMP(TopDocs *self, OutStream *outstream) { TopDocsIVARS *const ivars = TopDocs_IVARS(self); Freezer_serialize_varray(ivars->match_docs, outstream); OutStream_Write_C32(outstream, ivars->total_hits); }
void Freezer_serialize_blob(Blob *blob, OutStream *outstream) { size_t size = Blob_Get_Size(blob); OutStream_Write_C32(outstream, size); OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size); }
void DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter, int32_t doc_id) { DocWriterIVARS *const ivars = DocWriter_IVARS(self); OutStream *dat_out = S_lazy_init(self); OutStream *ix_out = ivars->ix_out; uint32_t num_stored = 0; int64_t start = OutStream_Tell(dat_out); int64_t expected = OutStream_Tell(ix_out) / 8; // Verify doc id. if (doc_id != expected) { THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id); } // Write the number of stored fields. Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { FieldType *type = Inverter_Get_Type(inverter); if (FType_Stored(type)) { num_stored++; } } OutStream_Write_C32(dat_out, num_stored); Inverter_Iterate(inverter); while (Inverter_Next(inverter)) { // Only store fields marked as "stored". FieldType *type = Inverter_Get_Type(inverter); if (FType_Stored(type)) { String *field = Inverter_Get_Field_Name(inverter); Obj *value = Inverter_Get_Value(inverter); Freezer_serialize_string(field, dat_out); switch (FType_Primitive_ID(type) & FType_PRIMITIVE_ID_MASK) { case FType_TEXT: { const char *buf = Str_Get_Ptr8((String*)value); size_t size = Str_Get_Size((String*)value); OutStream_Write_C32(dat_out, size); OutStream_Write_Bytes(dat_out, buf, size); break; } case FType_BLOB: { char *buf = BB_Get_Buf((ByteBuf*)value); size_t size = BB_Get_Size((ByteBuf*)value); OutStream_Write_C32(dat_out, size); OutStream_Write_Bytes(dat_out, buf, size); break; } case FType_INT32: { int32_t val = Int32_Get_Value((Integer32*)value); OutStream_Write_C32(dat_out, val); break; } case FType_INT64: { int64_t val = Int64_Get_Value((Integer64*)value); OutStream_Write_C64(dat_out, val); break; } case FType_FLOAT32: { float val = Float32_Get_Value((Float32*)value); OutStream_Write_F32(dat_out, val); break; } case FType_FLOAT64: { double val = Float64_Get_Value((Float64*)value); OutStream_Write_F64(dat_out, val); break; } default: THROW(ERR, "Unrecognized type: %o", type); } } } // Write file pointer. OutStream_Write_I64(ix_out, start); }
void TopDocs_serialize(TopDocs *self, OutStream *outstream) { VA_Serialize(self->match_docs, outstream); OutStream_Write_C32(outstream, self->total_hits); }