DefaultHighlightReader*
DefHLReader_init(DefaultHighlightReader *self, Schema *schema,
Folder *folder, Snapshot *snapshot, Vector *segments,
int32_t seg_tick) {
HLReader_init((HighlightReader*)self, schema, folder, snapshot,
segments, seg_tick);
DefaultHighlightReaderIVARS *const ivars = DefHLReader_IVARS(self);
Segment *segment = DefHLReader_Get_Segment(self);
Hash *metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "highlight", 9);
if (!metadata) {
metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "term_vectors", 12);
}
// Check format.
if (metadata) {
Obj *format = Hash_Fetch_Utf8(metadata, "format", 6);
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
if (Json_obj_to_i64(format) != HLWriter_current_file_format) {
THROW(ERR, "Unsupported highlight data format: %i64",
Json_obj_to_i64(format));
}
}
}
// Open instreams.
String *seg_name = Seg_Get_Name(segment);
String *ix_file = Str_newf("%o/highlight.ix", seg_name);
String *dat_file = Str_newf("%o/highlight.dat", seg_name);
if (Folder_Exists(folder, ix_file)) {
ivars->ix_in = Folder_Open_In(folder, ix_file);
if (!ivars->ix_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
ivars->dat_in = Folder_Open_In(folder, dat_file);
if (!ivars->dat_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
}
DECREF(ix_file);
DECREF(dat_file);
return self;
}
static void
test_Write_File_and_Read_File(TestBatchRunner *runner) {
RAMFolder *folder = RAMFolder_new(NULL);
Segment *segment = Seg_new(100);
Segment *got = Seg_new(100);
String *meta;
String *flotsam = (String*)SSTR_WRAP_UTF8("flotsam", 7);
String *jetsam = (String*)SSTR_WRAP_UTF8("jetsam", 6);
Seg_Set_Count(segment, 111);
Seg_Store_Metadata_Utf8(segment, "foo", 3, (Obj*)Str_newf("bar"));
Seg_Add_Field(segment, flotsam);
Seg_Add_Field(segment, jetsam);
RAMFolder_MkDir(folder, Seg_Get_Name(segment));
Seg_Write_File(segment, (Folder*)folder);
Seg_Read_File(got, (Folder*)folder);
TEST_TRUE(runner, Seg_Get_Count(got) == Seg_Get_Count(segment),
"Round-trip count through file");
TEST_TRUE(runner,
Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam),
"Round trip field names through file");
meta = (String*)Seg_Fetch_Metadata_Utf8(got, "foo", 3);
TEST_TRUE(runner,
meta
&& Str_Is_A(meta, STRING)
&& Str_Equals_Utf8(meta, "bar", 3),
"Round trip metadata through file");
DECREF(got);
DECREF(segment);
DECREF(folder);
}
DefaultSortReader*
DefSortReader_init(DefaultSortReader *self, Schema *schema, Folder *folder,
Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
seg_tick);
DefaultSortReaderIVARS *const ivars = DefSortReader_IVARS(self);
Segment *segment = DefSortReader_Get_Segment(self);
Hash *metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "sort", 4);
// Check format.
ivars->format = 0;
if (metadata) {
Obj *format = Hash_Fetch_Utf8(metadata, "format", 6);
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
ivars->format = (int32_t)Obj_To_I64(format);
if (ivars->format < 2 || ivars->format > 3) {
THROW(ERR, "Unsupported sort cache format: %i32",
ivars->format);
}
}
}
// Init.
ivars->caches = Hash_new(0);
// Either extract or fake up the "counts", "null_ords", and "ord_widths"
// hashes.
if (metadata) {
ivars->counts
= (Hash*)INCREF(CERTIFY(Hash_Fetch_Utf8(metadata, "counts", 6),
HASH));
ivars->null_ords = (Hash*)Hash_Fetch_Utf8(metadata, "null_ords", 9);
if (ivars->null_ords) {
ivars->null_ords = (Hash*)INCREF(CERTIFY(ivars->null_ords, HASH));
}
else {
ivars->null_ords = Hash_new(0);
}
ivars->ord_widths = (Hash*)Hash_Fetch_Utf8(metadata, "ord_widths", 10);
if (ivars->ord_widths) {
ivars->ord_widths
= (Hash*)INCREF(CERTIFY(ivars->ord_widths, HASH));
}
else {
ivars->ord_widths = Hash_new(0);
}
}
else {
ivars->counts = Hash_new(0);
ivars->null_ords = Hash_new(0);
ivars->ord_widths = Hash_new(0);
}
return self;
}
DefaultDocReader*
DefDocReader_init(DefaultDocReader *self, Schema *schema, Folder *folder,
Snapshot *snapshot, Vector *segments, int32_t seg_tick) {
Hash *metadata;
Segment *segment;
DocReader_init((DocReader*)self, schema, folder, snapshot, segments,
seg_tick);
DefaultDocReaderIVARS *const ivars = DefDocReader_IVARS(self);
segment = DefDocReader_Get_Segment(self);
metadata = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "documents", 9);
if (metadata) {
String *seg_name = Seg_Get_Name(segment);
String *ix_file = Str_newf("%o/documents.ix", seg_name);
String *dat_file = Str_newf("%o/documents.dat", seg_name);
Obj *format = Hash_Fetch_Utf8(metadata, "format", 6);
// Check format.
if (!format) { THROW(ERR, "Missing 'format' var"); }
else {
int64_t format_val = Json_obj_to_i64(format);
if (format_val < DocWriter_current_file_format) {
THROW(ERR, "Obsolete doc storage format %i64; "
"Index regeneration is required", format_val);
}
else if (format_val != DocWriter_current_file_format) {
THROW(ERR, "Unsupported doc storage format: %i64", format_val);
}
}
// Get streams.
if (Folder_Exists(folder, ix_file)) {
ivars->ix_in = Folder_Open_In(folder, ix_file);
if (!ivars->ix_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
ivars->dat_in = Folder_Open_In(folder, dat_file);
if (!ivars->dat_in) {
Err *error = (Err*)INCREF(Err_get_error());
DECREF(ix_file);
DECREF(dat_file);
DECREF(self);
RETHROW(error);
}
}
DECREF(ix_file);
DECREF(dat_file);
}
return self;
}
static void
test_metadata_storage(TestBatchRunner *runner) {
Segment *segment = Seg_new(1);
String *got;
Seg_Store_Metadata_Utf8(segment, "foo", 3, (Obj*)Str_newf("bar"));
got = (String*)Seg_Fetch_Metadata_Utf8(segment, "foo", 3);
TEST_TRUE(runner,
got
&& Str_Is_A(got, STRING)
&& Str_Equals_Utf8(got, "bar", 3),
"metadata round trip"
);
DECREF(segment);
}
void
DefDelWriter_Merge_Segment_IMP(DefaultDeletionsWriter *self,
SegReader *reader, I32Array *doc_map) {
DefaultDeletionsWriterIVARS *const ivars = DefDelWriter_IVARS(self);
UNUSED_VAR(doc_map);
Segment *segment = SegReader_Get_Segment(reader);
Hash *del_meta = (Hash*)Seg_Fetch_Metadata_Utf8(segment, "deletions", 9);
if (del_meta) {
Vector *seg_readers = ivars->seg_readers;
Hash *files = (Hash*)Hash_Fetch_Utf8(del_meta, "files", 5);
if (files) {
HashIterator *iter = HashIter_new(files);
while (HashIter_Next(iter)) {
String *seg = HashIter_Get_Key(iter);
Hash *mini_meta = (Hash*)HashIter_Get_Value(iter);
/* Find the segment the deletions from the SegReader
* we're adding correspond to. If it's gone, we don't
* need to worry about losing deletions files that point
* at it. */
for (size_t i = 0, max = Vec_Get_Size(seg_readers); i < max; i++) {
SegReader *candidate
= (SegReader*)Vec_Fetch(seg_readers, i);
String *candidate_name
= Seg_Get_Name(SegReader_Get_Segment(candidate));
if (Str_Equals(seg, (Obj*)candidate_name)) {
/* If the count hasn't changed, we're about to
* merge away the most recent deletions file
* pointing at this target segment -- so force a
* new file to be written out. */
int32_t count = (int32_t)Json_obj_to_i64(Hash_Fetch_Utf8(mini_meta, "count", 5));
DeletionsReader *del_reader
= (DeletionsReader*)SegReader_Obtain(
candidate, Class_Get_Name(DELETIONSREADER));
if (count == DelReader_Del_Count(del_reader)) {
ivars->updated[i] = true;
}
break;
}
}
}
DECREF(iter);
}
}
}