void
SortWriter_Add_Inverted_Doc_IMP(SortWriter *self, Inverter *inverter,
int32_t doc_id) {
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
int32_t field_num;
Inverter_Iterate(inverter);
while (0 != (field_num = Inverter_Next(inverter))) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Sortable(type)) {
SortFieldWriter *field_writer
= S_lazy_init_field_writer(self, field_num);
SortFieldWriter_Add(field_writer, doc_id,
Inverter_Get_Value(inverter));
}
}
// If our SortFieldWriters have collectively passed the memory threshold,
// flush all of them, then reset the counter which tracks memory
// consumption.
if ((size_t)Counter_Get_Value(ivars->counter) > ivars->mem_thresh) {
for (size_t i = 0; i < Vec_Get_Size(ivars->field_writers); i++) {
SortFieldWriter *const field_writer
= (SortFieldWriter*)Vec_Fetch(ivars->field_writers, i);
if (field_writer) { SortFieldWriter_Flush(field_writer); }
}
Counter_Reset(ivars->counter);
ivars->flush_at_finish = true;
}
}
void
SortWriter_add_inverted_doc(SortWriter *self, Inverter *inverter,
int32_t doc_id) {
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
int32_t field_num;
Inverter_Iterate(inverter);
while (0 != (field_num = Inverter_Next(inverter))) {
FieldType *type = Inverter_Get_Type(inverter);
if (FType_Sortable(type)) {
SortFieldWriter *field_writer
= S_lazy_init_field_writer(self, field_num);
SortFieldWriter_Add(field_writer, doc_id,
Inverter_Get_Value(inverter));
}
}
// If our SortFieldWriters have collectively passed the memory threshold,
// flush all of them, then release all unique values with a single action.
if (MemPool_Get_Consumed(ivars->mem_pool) > ivars->mem_thresh) {
for (uint32_t i = 0; i < VA_Get_Size(ivars->field_writers); i++) {
SortFieldWriter *const field_writer
= (SortFieldWriter*)VA_Fetch(ivars->field_writers, i);
if (field_writer) { SortFieldWriter_Flush(field_writer); }
}
MemPool_Release_All(ivars->mem_pool);
ivars->flush_at_finish = true;
}
}
uint32_t
SortFieldWriter_Refill_IMP(SortFieldWriter *self) {
SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
if (!ivars->sort_cache) { return 0; }
// Sanity check, then reset the buffer and prepare to start loading items.
uint32_t buf_count = SortFieldWriter_Buffer_Count(self);
if (buf_count) {
THROW(ERR, "Refill called but buffer contains %u32 items",
buf_count);
}
SortFieldWriter_Clear_Buffer(self);
Counter_Reset(ivars->counter);
S_lazy_init_sorted_ids(self);
const int32_t null_ord = ivars->null_ord;
I32Array *const doc_map = ivars->doc_map;
SortCache *const sort_cache = ivars->sort_cache;
uint32_t count = 0;
while (ivars->run_tick <= ivars->run_max
&& Counter_Get_Value(ivars->counter) < ivars->mem_thresh
) {
int32_t raw_doc_id = ivars->sorted_ids[ivars->run_tick];
int32_t ord = SortCache_Ordinal(sort_cache, raw_doc_id);
if (ord != null_ord) {
int32_t remapped = doc_map
? I32Arr_Get(doc_map, raw_doc_id)
: raw_doc_id;
if (remapped) {
Obj *val = SortCache_Value(sort_cache, ord);
SortFieldWriter_Add(self, remapped, val);
count++;
DECREF(val);
}
}
ivars->run_tick++;
}
if (ivars->run_tick > ivars->run_max) {
DECREF(ivars->sort_cache);
ivars->sort_cache = NULL;
FREEMEM(ivars->sorted_ids);
ivars->sorted_ids = NULL;
}
return count;
}
