void
SortWriter_add_inverted_doc(SortWriter *self, Inverter *inverter,
int32_t doc_id) {
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
int32_t field_num;
Inverter_Iterate(inverter);
while (0 != (field_num = Inverter_Next(inverter))) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Sortable(type)) {
SortFieldWriter *field_writer
= S_lazy_init_field_writer(self, field_num);
SortFieldWriter_Add(field_writer, doc_id,
Inverter_Get_Value(inverter));
}
}
// If our SortFieldWriters have collectively passed the memory threshold,
// flush all of them, then release all unique values with a single action.
if (MemPool_Get_Consumed(ivars->mem_pool) > ivars->mem_thresh) {
for (uint32_t i = 0; i < VA_Get_Size(ivars->field_writers); i++) {
SortFieldWriter *const field_writer
= (SortFieldWriter*)VA_Fetch(ivars->field_writers, i);
if (field_writer) { SortFieldWriter_Flush(field_writer); }
}
MemPool_Release_All(ivars->mem_pool);
ivars->flush_at_finish = true;
}
}
void
SortWriter_Add_Inverted_Doc_IMP(SortWriter *self, Inverter *inverter,
int32_t doc_id) {
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
int32_t field_num;
Inverter_Iterate(inverter);
while (0 != (field_num = Inverter_Next(inverter))) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Sortable(type)) {
SortFieldWriter *field_writer
= S_lazy_init_field_writer(self, field_num);
SortFieldWriter_Add(field_writer, doc_id,
Inverter_Get_Value(inverter));
}
}
// If our SortFieldWriters have collectively passed the memory threshold,
// flush all of them, then reset the counter which tracks memory
// consumption.
if ((size_t)Counter_Get_Value(ivars->counter) > ivars->mem_thresh) {
for (size_t i = 0; i < Vec_Get_Size(ivars->field_writers); i++) {
SortFieldWriter *const field_writer
= (SortFieldWriter*)Vec_Fetch(ivars->field_writers, i);
if (field_writer) { SortFieldWriter_Flush(field_writer); }
}
Counter_Reset(ivars->counter);
ivars->flush_at_finish = true;
}
}
void
DocWriter_add_inverted_doc(DocWriter *self, Inverter *inverter,
int32_t doc_id) {
OutStream *dat_out = S_lazy_init(self);
OutStream *ix_out = self->ix_out;
uint32_t num_stored = 0;
int64_t start = OutStream_Tell(dat_out);
int64_t expected = OutStream_Tell(ix_out) / 8;
// Verify doc id.
if (doc_id != expected) {
THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id);
}
// Write the number of stored fields.
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) { num_stored++; }
}
OutStream_Write_C32(dat_out, num_stored);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
// Only store fields marked as "stored".
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) {
CharBuf *field = Inverter_Get_Field_Name(inverter);
Obj *value = Inverter_Get_Value(inverter);
CB_Serialize(field, dat_out);
Obj_Serialize(value, dat_out);
}
}
// Write file pointer.
OutStream_Write_I64(ix_out, start);
}
void
DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter,
int32_t doc_id) {
DocWriterIVARS *const ivars = DocWriter_IVARS(self);
OutStream *dat_out = S_lazy_init(self);
OutStream *ix_out = ivars->ix_out;
uint32_t num_stored = 0;
int64_t start = OutStream_Tell(dat_out);
int64_t expected = OutStream_Tell(ix_out) / 8;
// Verify doc id.
if (doc_id != expected) {
THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id);
}
// Write the number of stored fields.
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) { num_stored++; }
}
OutStream_Write_C32(dat_out, num_stored);
Inverter_Iterate(inverter);
while (Inverter_Next(inverter)) {
// Only store fields marked as "stored".
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Stored(type)) {
String *field = Inverter_Get_Field_Name(inverter);
Obj *value = Inverter_Get_Value(inverter);
Freezer_serialize_string(field, dat_out);
switch (FType_Primitive_ID(type) & FType_PRIMITIVE_ID_MASK) {
case FType_TEXT: {
const char *buf = Str_Get_Ptr8((String*)value);
size_t size = Str_Get_Size((String*)value);
OutStream_Write_C32(dat_out, size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
case FType_BLOB: {
char *buf = BB_Get_Buf((ByteBuf*)value);
size_t size = BB_Get_Size((ByteBuf*)value);
OutStream_Write_C32(dat_out, size);
OutStream_Write_Bytes(dat_out, buf, size);
break;
}
case FType_INT32: {
int32_t val = Int32_Get_Value((Integer32*)value);
OutStream_Write_C32(dat_out, val);
break;
}
case FType_INT64: {
int64_t val = Int64_Get_Value((Integer64*)value);
OutStream_Write_C64(dat_out, val);
break;
}
case FType_FLOAT32: {
float val = Float32_Get_Value((Float32*)value);
OutStream_Write_F32(dat_out, val);
break;
}
case FType_FLOAT64: {
double val = Float64_Get_Value((Float64*)value);
OutStream_Write_F64(dat_out, val);
break;
}
default:
THROW(ERR, "Unrecognized type: %o", type);
}
}
}
// Write file pointer.
OutStream_Write_I64(ix_out, start);
}
