void SortWriter_Add_Inverted_Doc_IMP(SortWriter *self, Inverter *inverter, int32_t doc_id) { SortWriterIVARS *const ivars = SortWriter_IVARS(self); int32_t field_num; Inverter_Iterate(inverter); while (0 != (field_num = Inverter_Next(inverter))) { FieldType *type = Inverter_Get_Type(inverter); if (FType_Sortable(type)) { SortFieldWriter *field_writer = S_lazy_init_field_writer(self, field_num); SortFieldWriter_Add(field_writer, doc_id, Inverter_Get_Value(inverter)); } } // If our SortFieldWriters have collectively passed the memory threshold, // flush all of them, then reset the counter which tracks memory // consumption. if ((size_t)Counter_Get_Value(ivars->counter) > ivars->mem_thresh) { for (size_t i = 0; i < Vec_Get_Size(ivars->field_writers); i++) { SortFieldWriter *const field_writer = (SortFieldWriter*)Vec_Fetch(ivars->field_writers, i); if (field_writer) { SortFieldWriter_Flush(field_writer); } } Counter_Reset(ivars->counter); ivars->flush_at_finish = true; } }
uint32_t SortFieldWriter_Refill_IMP(SortFieldWriter *self) { SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self); if (!ivars->sort_cache) { return 0; } // Sanity check, then reset the buffer and prepare to start loading items. uint32_t buf_count = SortFieldWriter_Buffer_Count(self); if (buf_count) { THROW(ERR, "Refill called but buffer contains %u32 items", buf_count); } SortFieldWriter_Clear_Buffer(self); Counter_Reset(ivars->counter); S_lazy_init_sorted_ids(self); const int32_t null_ord = ivars->null_ord; I32Array *const doc_map = ivars->doc_map; SortCache *const sort_cache = ivars->sort_cache; uint32_t count = 0; while (ivars->run_tick <= ivars->run_max && Counter_Get_Value(ivars->counter) < ivars->mem_thresh ) { int32_t raw_doc_id = ivars->sorted_ids[ivars->run_tick]; int32_t ord = SortCache_Ordinal(sort_cache, raw_doc_id); if (ord != null_ord) { int32_t remapped = doc_map ? I32Arr_Get(doc_map, raw_doc_id) : raw_doc_id; if (remapped) { Obj *val = SortCache_Value(sort_cache, ord); SortFieldWriter_Add(self, remapped, val); count++; DECREF(val); } } ivars->run_tick++; } if (ivars->run_tick > ivars->run_max) { DECREF(ivars->sort_cache); ivars->sort_cache = NULL; FREEMEM(ivars->sorted_ids); ivars->sorted_ids = NULL; } return count; }