void
DefDelWriter_finish(DefaultDeletionsWriter *self) {
DefaultDeletionsWriterIVARS *const ivars = DefDelWriter_IVARS(self);
Folder *const folder = ivars->folder;
for (uint32_t i = 0, max = VA_Get_Size(ivars->seg_readers); i < max; i++) {
SegReader *seg_reader = (SegReader*)VA_Fetch(ivars->seg_readers, i);
if (ivars->updated[i]) {
BitVector *deldocs = (BitVector*)VA_Fetch(ivars->bit_vecs, i);
int32_t doc_max = SegReader_Doc_Max(seg_reader);
double used = (doc_max + 1) / 8.0;
uint32_t byte_size = (uint32_t)ceil(used);
uint32_t new_max = byte_size * 8 - 1;
CharBuf *filename = S_del_filename(self, seg_reader);
OutStream *outstream = Folder_Open_Out(folder, filename);
if (!outstream) { RETHROW(INCREF(Err_get_error())); }
// Ensure that we have 1 bit for each doc in segment.
BitVec_Grow(deldocs, new_max);
// Write deletions data and clean up.
OutStream_Write_Bytes(outstream,
(char*)BitVec_Get_Raw_Bits(deldocs),
byte_size);
OutStream_Close(outstream);
DECREF(outstream);
DECREF(filename);
}
}
Seg_Store_Metadata_Str(ivars->segment, "deletions", 9,
(Obj*)DefDelWriter_Metadata(self));
}
static void
test_Write_File_and_Read_File(TestBatch *batch)
{
RAMFolder *folder = RAMFolder_new(NULL);
Segment *segment = Seg_new(100);
Segment *got = Seg_new(100);
CharBuf *meta;
CharBuf *flotsam = (CharBuf*)ZCB_WRAP_STR("flotsam", 7);
CharBuf *jetsam = (CharBuf*)ZCB_WRAP_STR("jetsam", 6);
Seg_Set_Count(segment, 111);
Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
Seg_Add_Field(segment, flotsam);
Seg_Add_Field(segment, jetsam);
RAMFolder_MkDir(folder, Seg_Get_Name(segment));
Seg_Write_File(segment, (Folder*)folder);
Seg_Read_File(got, (Folder*)folder);
TEST_TRUE(batch, Seg_Get_Count(got) == Seg_Get_Count(segment),
"Round-trip count through file");
TEST_TRUE(batch,
Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam),
"Round trip field names through file");
meta = (CharBuf*)Seg_Fetch_Metadata_Str(got, "foo", 3);
TEST_TRUE(batch, meta && CB_Is_A(meta, CHARBUF)
&& CB_Equals_Str(meta, "bar", 3), "Round trip metadata through file");
DECREF(got);
DECREF(segment);
DECREF(folder);
}
static void
test_metadata_storage(TestBatch *batch)
{
Segment *segment = Seg_new(1);
CharBuf *got;
Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
got = (CharBuf*)Seg_Fetch_Metadata_Str(segment, "foo", 3);
TEST_TRUE(batch,
got
&& CB_Is_A(got, CHARBUF)
&& CB_Equals_Str(got, "bar", 3),
"metadata round trip");
DECREF(segment);
}
void
DocWriter_finish(DocWriter *self) {
if (self->dat_out) {
// Write one final file pointer, so that we can derive the length of
// the last record.
int64_t end = OutStream_Tell(self->dat_out);
OutStream_Write_I64(self->ix_out, end);
// Close down output streams.
OutStream_Close(self->dat_out);
OutStream_Close(self->ix_out);
Seg_Store_Metadata_Str(self->segment, "documents", 9,
(Obj*)DocWriter_Metadata(self));
}
}
void
HLWriter_finish(HighlightWriter *self) {
HighlightWriterIVARS *const ivars = HLWriter_IVARS(self);
if (ivars->dat_out) {
// Write one final file pointer, so that we can derive the length of
// the last record.
int64_t end = OutStream_Tell(ivars->dat_out);
OutStream_Write_I64(ivars->ix_out, end);
// Close down the output streams.
OutStream_Close(ivars->dat_out);
OutStream_Close(ivars->ix_out);
Seg_Store_Metadata_Str(ivars->segment, "highlight", 9,
(Obj*)HLWriter_Metadata(self));
}
}
void
HLWriter_finish(HighlightWriter *self)
{
if (self->dat_out) {
/* Write one final file pointer, so that we can derive the length of
* the last record. */
i64_t end = OutStream_Tell(self->dat_out);
OutStream_Write_U64(self->ix_out, end);
/* Close down the output streams. */
OutStream_Close(self->dat_out);
OutStream_Close(self->ix_out);
Seg_Store_Metadata_Str(self->segment, "highlight", 9,
(Obj*)HLWriter_Metadata(self));
}
}
void
LexWriter_finish(LexiconWriter *self)
{
/* Ensure that streams were closed (by calling Finish_Field or
* Leave_Temp_Mode). */
if (self->dat_out != NULL) {
THROW("File '%o' never closed", self->dat_file);
}
else if (self->ix_out != NULL) {
THROW("File '%o' never closed", self->ix_file);
}
else if (self->ix_out != NULL) {
THROW("File '%o' never closed", self->ix_file);
}
/* Store metadata. */
Seg_Store_Metadata_Str(self->segment, "lexicon", 7,
(Obj*)LexWriter_Metadata(self));
}
void
SortWriter_finish(SortWriter *self) {
SortWriterIVARS *const ivars = SortWriter_IVARS(self);
VArray *const field_writers = ivars->field_writers;
// If we have no data, bail out.
if (!ivars->temp_ord_out) { return; }
// If we've either flushed or added segments, flush everything so that any
// one field can use the entire margin up to mem_thresh.
if (ivars->flush_at_finish) {
for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) {
SortFieldWriter *field_writer
= (SortFieldWriter*)VA_Fetch(field_writers, i);
if (field_writer) {
SortFieldWriter_Flush(field_writer);
}
}
}
// Close down temp streams.
OutStream_Close(ivars->temp_ord_out);
OutStream_Close(ivars->temp_ix_out);
OutStream_Close(ivars->temp_dat_out);
for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) {
SortFieldWriter *field_writer
= (SortFieldWriter*)VA_Delete(field_writers, i);
if (field_writer) {
CharBuf *field = Seg_Field_Name(ivars->segment, i);
SortFieldWriter_Flip(field_writer);
int32_t count = SortFieldWriter_Finish(field_writer);
Hash_Store(ivars->counts, (Obj*)field,
(Obj*)CB_newf("%i32", count));
int32_t null_ord = SortFieldWriter_Get_Null_Ord(field_writer);
if (null_ord != -1) {
Hash_Store(ivars->null_ords, (Obj*)field,
(Obj*)CB_newf("%i32", null_ord));
}
int32_t ord_width = SortFieldWriter_Get_Ord_Width(field_writer);
Hash_Store(ivars->ord_widths, (Obj*)field,
(Obj*)CB_newf("%i32", ord_width));
}
DECREF(field_writer);
}
VA_Clear(field_writers);
// Store metadata.
Seg_Store_Metadata_Str(ivars->segment, "sort", 4,
(Obj*)SortWriter_Metadata(self));
// Clean up.
Folder *folder = ivars->folder;
CharBuf *seg_name = Seg_Get_Name(ivars->segment);
CharBuf *path = CB_newf("%o/sort_ord_temp", seg_name);
Folder_Delete(folder, path);
CB_setf(path, "%o/sort_ix_temp", seg_name);
Folder_Delete(folder, path);
CB_setf(path, "%o/sort_dat_temp", seg_name);
Folder_Delete(folder, path);
DECREF(path);
}
