DefaultHighlightReader*
DefHLReader_init(DefaultHighlightReader *self, Schema *schema,
Folder *folder, Snapshot *snapshot, VArray *segments,
int32_t seg_tick)
{
Segment *segment;
Hash *metadata;
HLReader_init((HighlightReader*)self, schema, folder, snapshot,
segments, seg_tick);
segment = DefHLReader_Get_Segment(self);
metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "highlight", 9);
if (!metadata) {
metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "term_vectors", 12);
}
// Check format.
if (metadata) {
Obj *format = Hash_Fetch_Str(metadata, "format", 6);
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
if (Obj_To_I64(format) != HLWriter_current_file_format) {
THROW(ERR, "Unsupported highlight data format: %i64",
Obj_To_I64(format));
}
}
}
// Open instreams.
{
CharBuf *seg_name = Seg_Get_Name(segment);
CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
if (Folder_Exists(folder, ix_file)) {
self->ix_in = Folder_Open_In(folder, ix_file);
if (!self->ix_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
self->dat_in = Folder_Open_In(folder, dat_file);
if (!self->dat_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
}
DECREF(ix_file);
DECREF(dat_file);
}
return self;
}
static void
test_Write_File_and_Read_File(TestBatch *batch)
{
RAMFolder *folder = RAMFolder_new(NULL);
Segment *segment = Seg_new(100);
Segment *got = Seg_new(100);
CharBuf *meta;
CharBuf *flotsam = (CharBuf*)ZCB_WRAP_STR("flotsam", 7);
CharBuf *jetsam = (CharBuf*)ZCB_WRAP_STR("jetsam", 6);
Seg_Set_Count(segment, 111);
Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
Seg_Add_Field(segment, flotsam);
Seg_Add_Field(segment, jetsam);
RAMFolder_MkDir(folder, Seg_Get_Name(segment));
Seg_Write_File(segment, (Folder*)folder);
Seg_Read_File(got, (Folder*)folder);
TEST_TRUE(batch, Seg_Get_Count(got) == Seg_Get_Count(segment),
"Round-trip count through file");
TEST_TRUE(batch,
Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam),
"Round trip field names through file");
meta = (CharBuf*)Seg_Fetch_Metadata_Str(got, "foo", 3);
TEST_TRUE(batch, meta && CB_Is_A(meta, CHARBUF)
&& CB_Equals_Str(meta, "bar", 3), "Round trip metadata through file");
DECREF(got);
DECREF(segment);
DECREF(folder);
}
DefaultDocReader*
DefDocReader_init(DefaultDocReader *self, Schema *schema, Folder *folder,
Snapshot *snapshot, VArray *segments, i32_t seg_tick)
{
Hash *metadata;
Segment *segment;
DocReader_init((DocReader*)self, schema, folder, snapshot, segments,
seg_tick);
segment = DefDocReader_Get_Segment(self);
metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "documents", 9);
if (metadata) {
CharBuf *seg_name = Seg_Get_Name(segment);
CharBuf *ix_file = CB_newf("%o/documents.ix", seg_name);
CharBuf *dat_file = CB_newf("%o/documents.dat", seg_name);
Obj *format = Hash_Fetch_Str(metadata, "format", 6);
/* Check format. */
if (!format) { THROW("Missing 'format' var"); }
else {
i64_t format_val = Obj_To_I64(format);
if (format_val < DocWriter_current_file_format) {
THROW("Obsolete doc storage format %i64; "
"Index regeneration is required", format_val);
}
else if (format_val != DocWriter_current_file_format) {
THROW("Unsupported doc storage format: %i64", format_val);
}
}
/* Get streams. */
if (Folder_Exists(folder, ix_file)) {
self->ix_in = Folder_Open_In(folder, ix_file);
self->dat_in = Folder_Open_In(folder, dat_file);
if (!self->ix_in || !self->dat_in) {
CharBuf *mess = MAKE_MESS("Can't open either %o or %o",
ix_file, dat_file);
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
Err_throw_mess(mess);
}
}
DECREF(ix_file);
DECREF(dat_file);
}
return self;
}
static void
test_metadata_storage(TestBatch *batch)
{
Segment *segment = Seg_new(1);
CharBuf *got;
Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
got = (CharBuf*)Seg_Fetch_Metadata_Str(segment, "foo", 3);
TEST_TRUE(batch,
got
&& CB_Is_A(got, CHARBUF)
&& CB_Equals_Str(got, "bar", 3),
"metadata round trip");
DECREF(segment);
}
void
DefDelWriter_merge_segment(DefaultDeletionsWriter *self, SegReader *reader,
I32Array *doc_map) {
DefaultDeletionsWriterIVARS *const ivars = DefDelWriter_IVARS(self);
UNUSED_VAR(doc_map);
Segment *segment = SegReader_Get_Segment(reader);
Hash *del_meta = (Hash*)Seg_Fetch_Metadata_Str(segment, "deletions", 9);
if (del_meta) {
VArray *seg_readers = ivars->seg_readers;
Hash *files = (Hash*)Hash_Fetch_Str(del_meta, "files", 5);
if (files) {
CharBuf *seg;
Hash *mini_meta;
Hash_Iterate(files);
while (Hash_Next(files, (Obj**)&seg, (Obj**)&mini_meta)) {
/* Find the segment the deletions from the SegReader
* we're adding correspond to. If it's gone, we don't
* need to worry about losing deletions files that point
* at it. */
for (uint32_t i = 0, max = VA_Get_Size(seg_readers); i < max; i++) {
SegReader *candidate
= (SegReader*)VA_Fetch(seg_readers, i);
CharBuf *candidate_name
= Seg_Get_Name(SegReader_Get_Segment(candidate));
if (CB_Equals(seg, (Obj*)candidate_name)) {
/* If the count hasn't changed, we're about to
* merge away the most recent deletions file
* pointing at this target segment -- so force a
* new file to be written out. */
int32_t count = (int32_t)Obj_To_I64(Hash_Fetch_Str(mini_meta, "count", 5));
DeletionsReader *del_reader
= (DeletionsReader*)SegReader_Obtain(
candidate, VTable_Get_Name(DELETIONSREADER));
if (count == DelReader_Del_Count(del_reader)) {
ivars->updated[i] = true;
}
break;
}
}
}
}
}
}
