static void
test_Write_File_and_Read_File(TestBatchRunner *runner) {
RAMFolder *folder = RAMFolder_new(NULL);
Segment *segment = Seg_new(100);
Segment *got = Seg_new(100);
String *meta;
String *flotsam = (String*)SSTR_WRAP_UTF8("flotsam", 7);
String *jetsam = (String*)SSTR_WRAP_UTF8("jetsam", 6);
Seg_Set_Count(segment, 111);
Seg_Store_Metadata_Utf8(segment, "foo", 3, (Obj*)Str_newf("bar"));
Seg_Add_Field(segment, flotsam);
Seg_Add_Field(segment, jetsam);
RAMFolder_MkDir(folder, Seg_Get_Name(segment));
Seg_Write_File(segment, (Folder*)folder);
Seg_Read_File(got, (Folder*)folder);
TEST_TRUE(runner, Seg_Get_Count(got) == Seg_Get_Count(segment),
"Round-trip count through file");
TEST_TRUE(runner,
Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam),
"Round trip field names through file");
meta = (String*)Seg_Fetch_Metadata_Utf8(got, "foo", 3);
TEST_TRUE(runner,
meta
&& Str_Is_A(meta, STRING)
&& Str_Equals_Utf8(meta, "bar", 3),
"Round trip metadata through file");
DECREF(got);
DECREF(segment);
DECREF(folder);
}
static void
test_fields(TestBatchRunner *runner) {
Segment *segment = Seg_new(1);
StackString *foo = SSTR_WRAP_UTF8("foo", 3);
StackString *bar = SSTR_WRAP_UTF8("bar", 3);
StackString *baz = SSTR_WRAP_UTF8("baz", 3);
int32_t field_num;
field_num = Seg_Add_Field(segment, (String*)foo);
TEST_TRUE(runner, field_num == 1,
"Add_Field returns field number, and field numbers start at 1");
field_num = Seg_Add_Field(segment, (String*)bar);
TEST_TRUE(runner, field_num == 2, "add a second field");
field_num = Seg_Add_Field(segment, (String*)foo);
TEST_TRUE(runner, field_num == 1,
"Add_Field returns existing field number if field is already known");
TEST_TRUE(runner, SStr_Equals(bar, (Obj*)Seg_Field_Name(segment, 2)),
"Field_Name");
TEST_TRUE(runner, Seg_Field_Name(segment, 3) == NULL,
"Field_Name returns NULL for unknown field number");
TEST_TRUE(runner, Seg_Field_Num(segment, (String*)bar) == 2,
"Field_Num");
TEST_TRUE(runner, Seg_Field_Num(segment, (String*)baz) == 0,
"Field_Num returns 0 for unknown field name");
DECREF(segment);
}
static void
test_Read_File_and_Write_File(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
Folder *folder = (Folder*)RAMFolder_new(NULL);
String *snap = (String*)SSTR_WRAP_UTF8("snap", 4);
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Write_File(snapshot, folder, snap);
Snapshot *dupe = Snapshot_new();
Snapshot *read_retval = Snapshot_Read_File(dupe, folder, snap);
TEST_TRUE(runner, dupe == read_retval, "Read_File() returns the object");
Vector *orig_list = Snapshot_List(snapshot);
Vector *dupe_list = Snapshot_List(dupe);
TEST_TRUE(runner, Vec_Equals(orig_list, (Obj*)dupe_list),
"Round trip through Write_File() and Read_File()");
DECREF(orig_list);
DECREF(dupe_list);
DECREF(dupe);
DECREF(snapshot);
DECREF(folder);
}
static void
test_Dump_Load_and_Equals(TestBatchRunner *runner) {
String *EN = (String*)SSTR_WRAP_UTF8("en", 2);
String *ES = (String*)SSTR_WRAP_UTF8("es", 2);
SnowballStemmer *stemmer = SnowStemmer_new(EN);
SnowballStemmer *other = SnowStemmer_new(ES);
Obj *dump = (Obj*)SnowStemmer_Dump(stemmer);
Obj *other_dump = (Obj*)SnowStemmer_Dump(other);
SnowballStemmer *clone = (SnowballStemmer*)SnowStemmer_Load(other, dump);
SnowballStemmer *other_clone = (SnowballStemmer*)SnowStemmer_Load(other, other_dump);
TEST_FALSE(runner,
SnowStemmer_Equals(stemmer, (Obj*)other),
"Equals() false with different language");
TEST_TRUE(runner,
SnowStemmer_Equals(stemmer, (Obj*)clone),
"Dump => Load round trip");
TEST_TRUE(runner,
SnowStemmer_Equals(other, (Obj*)other_clone),
"Dump => Load round trip");
DECREF(stemmer);
DECREF(dump);
DECREF(clone);
DECREF(other);
DECREF(other_dump);
DECREF(other_clone);
}
bool
CFReader_Local_Delete_IMP(CompoundFileReader *self, String *name) {
CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self);
Hash *record = (Hash*)Hash_Delete(ivars->records, name);
DECREF(record);
if (record == NULL) {
return Folder_Local_Delete(ivars->real_folder, name);
}
else {
// Once the number of virtual files falls to 0, remove the compound
// files.
if (Hash_Get_Size(ivars->records) == 0) {
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
if (!Folder_Delete(ivars->real_folder, cf_file)) {
return false;
}
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (!Folder_Delete(ivars->real_folder, cfmeta_file)) {
return false;
}
}
return true;
}
}
static void
test_Highlight_Excerpt(TestBatchRunner *runner, Searcher *searcher, Obj *query) {
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Highlighter *highlighter = Highlighter_new(searcher, query, content, 3);
String *highlighted;
Vector *spans = Vec_new(1);
Vec_Push(spans, (Obj*)Span_new(2, 1, 0.0f));
String *raw_excerpt = (String *)SSTR_WRAP_UTF8("a b c", 5);
highlighted = Highlighter_Highlight_Excerpt(highlighter, spans,
raw_excerpt, 0);
TEST_TRUE(runner,
Str_Equals_Utf8(highlighted, "a <strong>b</strong> c", 22),
"basic Highlight_Excerpt");
DECREF(highlighted);
DECREF(spans);
spans = Vec_new(2);
Vec_Push(spans, (Obj*)Span_new(0, 1, 1.0f));
Vec_Push(spans, (Obj*)Span_new(10, 10, 1.0f));
raw_excerpt = (String *)SSTR_WRAP_UTF8(PHI, 2);
highlighted = Highlighter_Highlight_Excerpt(highlighter, spans,
raw_excerpt, 0);
TEST_TRUE(runner,
Str_Equals_Utf8(highlighted, "<strong>Φ</strong>", 23),
"don't surround spans off end of raw excerpt.");
DECREF(highlighted);
DECREF(spans);
spans = Vec_new(1);
Vec_Push(spans, (Obj*)Span_new(3, 1, 1.0f));
raw_excerpt = (String *)SSTR_WRAP_UTF8(PHI " " PHI " " PHI, 8);
highlighted = Highlighter_Highlight_Excerpt(highlighter, spans,
raw_excerpt, 1);
TEST_TRUE(runner,
Str_Equals_Utf8(highlighted,
"Φ <strong>Φ</strong> Φ", 37),
"Highlight_Excerpt pays attention to offset");
DECREF(highlighted);
DECREF(spans);
spans = Vec_new(4);
Vec_Push(spans, (Obj*)Span_new(2, 10, 1.0f));
Vec_Push(spans, (Obj*)Span_new(2, 4, 1.0f));
Vec_Push(spans, (Obj*)Span_new(8, 9, 1.0f));
Vec_Push(spans, (Obj*)Span_new(8, 4, 1.0f));
raw_excerpt = (String *)SSTR_WRAP_UTF8(PHI " Oook. Urk. Ick. " PHI, 21);
highlighted = Highlighter_Highlight_Excerpt(highlighter, spans,
raw_excerpt, 0);
TEST_TRUE(runner,
Str_Equals_Utf8(highlighted,
"Φ <strong>Oook. Urk. Ick.</strong> Φ",
46),
"Highlight_Excerpt works with overlapping spans");
DECREF(highlighted);
DECREF(spans);
DECREF(highlighter);
}
static void
test_protect_symlinks(TestBatchRunner *runner) {
#ifdef ENABLE_SYMLINK_TESTS
FSFolder *folder = (FSFolder*)S_set_up();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
String *foo_boffo = (String*)SSTR_WRAP_UTF8("foo/boffo", 9);
FSFolder_MkDir(folder, foo);
FSFolder_MkDir(folder, bar);
OutStream *outstream = FSFolder_Open_Out(folder, foo_boffo);
DECREF(outstream);
if (!S_create_test_symlinks()) {
FAIL(runner, "symlink creation failed");
FAIL(runner, "symlink creation failed");
FAIL(runner, "symlink creation failed");
FAIL(runner, "symlink creation failed");
FAIL(runner, "symlink creation failed");
// Try to clean up anyway.
FSFolder_Delete_Tree(folder, foo);
FSFolder_Delete_Tree(folder, bar);
}
else {
VArray *list = FSFolder_List_R(folder, NULL);
bool saw_bazooka_boffo = false;
for (uint32_t i = 0, max = VA_Get_Size(list); i < max; i++) {
String *entry = (String*)VA_Fetch(list, i);
if (Str_Ends_With_Utf8(entry, "bazooka/boffo", 13)) {
saw_bazooka_boffo = true;
}
}
TEST_FALSE(runner, saw_bazooka_boffo,
"List_R() shouldn't follow symlinks");
DECREF(list);
TEST_TRUE(runner, FSFolder_Delete_Tree(folder, bar),
"Delete_Tree() returns true");
TEST_FALSE(runner, FSFolder_Exists(folder, bar),
"Tree is really gone");
TEST_TRUE(runner, FSFolder_Exists(folder, foo),
"Original folder sill there");
TEST_TRUE(runner, FSFolder_Exists(folder, foo_boffo),
"Delete_Tree() did not follow directory symlink");
FSFolder_Delete_Tree(folder, foo);
}
DECREF(folder);
S_tear_down();
#else
SKIP(runner, "Tests requiring symlink() disabled");
SKIP(runner, "Tests requiring symlink() disabled");
SKIP(runner, "Tests requiring symlink() disabled");
SKIP(runner, "Tests requiring symlink() disabled");
SKIP(runner, "Tests requiring symlink() disabled");
#endif // ENABLE_SYMLINK_TESTS
}
static void
test_hl_selection(TestBatchRunner *runner) {
Schema *schema = Schema_new();
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(plain_type, true);
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Schema_Spec_Field(schema, content, (FieldType*)plain_type);
DECREF(plain_type);
DECREF(tokenizer);
RAMFolder *folder = RAMFolder_new(NULL);
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
static char test_string[] =
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla NNN bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla MMM bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. "
"bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla. ";
Doc *doc = Doc_new(NULL, 0);
String *string = (String *)SSTR_WRAP_UTF8(test_string, strlen(test_string));
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
Indexer_Commit(indexer);
DECREF(indexer);
Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder);
Obj *query = (Obj*)SSTR_WRAP_UTF8("NNN MMM", 7);
Highlighter *highlighter = Highlighter_new(searcher, query, content, 200);
Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL);
HitDoc *hit = Hits_Next(hits);
String *excerpt = Highlighter_Create_Excerpt(highlighter, hit);
String *mmm = (String*)SSTR_WRAP_UTF8("MMM", 3);
String *nnn = (String*)SSTR_WRAP_UTF8("NNN", 3);
TEST_TRUE(runner, Str_Find(excerpt, mmm) >= 0 || Str_Find(excerpt, nnn) >= 0,
"Sentence boundary algo doesn't chop terms");
DECREF(excerpt);
DECREF(hit);
DECREF(hits);
DECREF(highlighter);
DECREF(searcher);
DECREF(folder);
DECREF(schema);
}
static void
S_test_extract_gen(TestBatchRunner *runner, const char *name, uint64_t gen,
const char *test_name) {
StackString *source = SSTR_WRAP_UTF8(name, strlen(name));
TEST_TRUE(runner, IxFileNames_extract_gen((String*)source) == gen,
test_name);
}
static FieldType*
S_alt_field_type() {
StackString *name = SSTR_WRAP_UTF8("DummyFieldType2", 15);
Class *klass = Class_singleton((String*)name, DUMMYFIELDTYPE);
FieldType *self = (FieldType*)Class_Make_Obj(klass);
return FType_init(self);
}
static void
S_test_local_part(TestBatchRunner *runner, const char *source,
const char *wanted, const char *test_name) {
StackString *source_str = SSTR_WRAP_UTF8(source, strlen(source));
String *got = IxFileNames_local_part((String*)source_str);
TEST_TRUE(runner, Str_Equals_Utf8(got, wanted, strlen(wanted)), test_name);
DECREF(got);
}
static void
test_Read_Write(TestBatchRunner *runner) {
FSFileHandle *fh;
const char *foo = "foo";
const char *bar = "bar";
char buffer[12];
char *buf = buffer;
String *test_filename = (String*)SSTR_WRAP_UTF8("_fstest", 7);
remove(Str_Get_Ptr8(test_filename));
fh = FSFH_open(test_filename,
FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
TEST_TRUE(runner, FSFH_Length(fh) == INT64_C(0), "Length initially 0");
TEST_TRUE(runner, FSFH_Write(fh, foo, 3), "Write returns success");
TEST_TRUE(runner, FSFH_Length(fh) == INT64_C(3), "Length after Write");
TEST_TRUE(runner, FSFH_Write(fh, bar, 3), "Write returns success");
TEST_TRUE(runner, FSFH_Length(fh) == INT64_C(6), "Length after 2 Writes");
Err_set_error(NULL);
TEST_FALSE(runner, FSFH_Read(fh, buf, 0, 2),
"Reading from a write-only handle returns false");
TEST_TRUE(runner, Err_get_error() != NULL,
"Reading from a write-only handle sets error");
if (!FSFH_Close(fh)) { RETHROW(INCREF(Err_get_error())); }
DECREF(fh);
// Reopen for reading.
Err_set_error(NULL);
fh = FSFH_open(test_filename, FH_READ_ONLY);
TEST_TRUE(runner, FSFH_Length(fh) == INT64_C(6), "Length on Read");
TEST_TRUE(runner, FSFH_Read(fh, buf, 0, 6), "Read returns success");
TEST_TRUE(runner, strncmp(buf, "foobar", 6) == 0, "Read/Write");
TEST_TRUE(runner, FSFH_Read(fh, buf, 2, 3), "Read returns success");
TEST_TRUE(runner, strncmp(buf, "oba", 3) == 0, "Read with offset");
Err_set_error(NULL);
TEST_FALSE(runner, FSFH_Read(fh, buf, -1, 4),
"Read() with a negative offset returns false");
TEST_TRUE(runner, Err_get_error() != NULL,
"Read() with a negative offset sets error");
Err_set_error(NULL);
TEST_FALSE(runner, FSFH_Read(fh, buf, 6, 1),
"Read() past EOF returns false");
TEST_TRUE(runner, Err_get_error() != NULL,
"Read() past EOF sets error");
Err_set_error(NULL);
TEST_FALSE(runner, FSFH_Write(fh, foo, 3),
"Writing to a read-only handle returns false");
TEST_TRUE(runner, Err_get_error() != NULL,
"Writing to a read-only handle sets error");
DECREF(fh);
remove(Str_Get_Ptr8(test_filename));
}
static void
test_Compare_Values(TestBatchRunner *runner) {
FieldType *type = (FieldType*)DummyFieldType_new();
StackString *a = SSTR_WRAP_UTF8("a", 1);
StackString *b = SSTR_WRAP_UTF8("b", 1);
TEST_TRUE(runner,
FType_Compare_Values(type, (Obj*)a, (Obj*)b) < 0,
"a less than b");
TEST_TRUE(runner,
FType_Compare_Values(type, (Obj*)b, (Obj*)a) > 0,
"b greater than a");
TEST_TRUE(runner,
FType_Compare_Values(type, (Obj*)b, (Obj*)b) == 0,
"b equals b");
DECREF(type);
}
static void
S_clean_up_old_temp_files(CompoundFileWriter *self,
CompoundFileWriterIVARS *ivars) {
UNUSED_VAR(self);
Folder *folder = ivars->folder;
String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
if (Folder_Exists(folder, cf_file)) {
if (!Folder_Delete(folder, cf_file)) {
THROW(ERR, "Can't delete '%o'", cf_file);
}
}
if (Folder_Exists(folder, cfmeta_temp)) {
if (!Folder_Delete(folder, cfmeta_temp)) {
THROW(ERR, "Can't delete '%o'", cfmeta_temp);
}
}
}
static void
test_highlighting(TestBatchRunner *runner) {
Schema *schema = Schema_new();
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *plain_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(plain_type, true);
FullTextType *dunked_type = FullTextType_new((Analyzer*)tokenizer);
FullTextType_Set_Highlightable(dunked_type, true);
FullTextType_Set_Boost(dunked_type, 0.1f);
String *content = (String*)SSTR_WRAP_UTF8("content", 7);
Schema_Spec_Field(schema, content, (FieldType*)plain_type);
String *alt = (String*)SSTR_WRAP_UTF8("alt", 3);
Schema_Spec_Field(schema, alt, (FieldType*)dunked_type);
DECREF(plain_type);
DECREF(dunked_type);
DECREF(tokenizer);
RAMFolder *folder = RAMFolder_new(NULL);
Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
Doc *doc = Doc_new(NULL, 0);
String *string = (String *)SSTR_WRAP_UTF8(TEST_STRING, TEST_STRING_LEN);
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
doc = Doc_new(NULL, 0);
string = (String *)SSTR_WRAP_UTF8("\"I see,\" said the blind man.", 28);
Doc_Store(doc, content, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
doc = Doc_new(NULL, 0);
string = (String *)SSTR_WRAP_UTF8("x but not why or 2ee", 20);
Doc_Store(doc, content, (Obj*)string);
string = (String *)SSTR_WRAP_UTF8(TEST_STRING
" and extra stuff so it scores lower",
TEST_STRING_LEN + 35);
Doc_Store(doc, alt, (Obj*)string);
Indexer_Add_Doc(indexer, doc, 1.0f);
DECREF(doc);
Indexer_Commit(indexer);
DECREF(indexer);
Searcher *searcher = (Searcher*)IxSearcher_new((Obj*)folder);
Obj *query = (Obj*)SSTR_WRAP_UTF8("\"x y z\" AND " PHI, 14);
Hits *hits = Searcher_Hits(searcher, query, 0, 10, NULL);
test_Raw_Excerpt(runner, searcher, query);
test_Highlight_Excerpt(runner, searcher, query);
test_Create_Excerpt(runner, searcher, query, hits);
DECREF(hits);
DECREF(searcher);
DECREF(folder);
DECREF(schema);
}
static Folder*
S_set_up() {
rmdir("_fstest");
String *test_dir = (String*)SSTR_WRAP_UTF8("_fstest", 7);
FSFolder *folder = FSFolder_new(test_dir);
FSFolder_Initialize(folder);
if (!FSFolder_Check(folder)) {
RETHROW(INCREF(Err_get_error()));
}
return (Folder*)folder;
}
static Obj*
S_new_testobj() {
StackString *class_name = SSTR_WRAP_UTF8("TestObj", 7);
Obj *obj;
Class *klass = Class_fetch_class((String*)class_name);
if (!klass) {
klass = Class_singleton((String*)class_name, OBJ);
}
obj = Class_Make_Obj(klass);
return Obj_init(obj);
}
static void
test_Compare_Values(TestBatchRunner *runner) {
StandardTokenizer *tokenizer = StandardTokenizer_new();
FullTextType *type = FullTextType_new((Analyzer*)tokenizer);
StackString *a = SSTR_WRAP_UTF8("a", 1);
StackString *b = SSTR_WRAP_UTF8("b", 1);
TEST_TRUE(runner,
FullTextType_Compare_Values(type, (Obj*)a, (Obj*)b) < 0,
"a less than b");
TEST_TRUE(runner,
FullTextType_Compare_Values(type, (Obj*)b, (Obj*)a) > 0,
"b greater than a");
TEST_TRUE(runner,
FullTextType_Compare_Values(type, (Obj*)b, (Obj*)b) == 0,
"b equals b");
DECREF(type);
DECREF(tokenizer);
}
void
test_disallow_updir(TestBatchRunner *runner) {
FSFolder *outer_folder = (FSFolder*)S_set_up();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
FSFolder_MkDir(outer_folder, foo);
FSFolder_MkDir(outer_folder, bar);
String *inner_path = (String*)SSTR_WRAP_UTF8("_fstest/foo", 11);
FSFolder *foo_folder = FSFolder_new(inner_path);
String *up_bar = (String*)SSTR_WRAP_UTF8("../bar", 6);
TEST_FALSE(runner, FSFolder_Exists(foo_folder, up_bar),
"up-dirs are inaccessible.");
DECREF(foo_folder);
FSFolder_Delete(outer_folder, foo);
FSFolder_Delete(outer_folder, bar);
DECREF(outer_folder);
S_tear_down();
}
static FileHandle*
S_new_filehandle() {
String *class_name = SSTR_WRAP_UTF8("TestFileHandle", 14);
FileHandle *fh;
Class *klass = Class_fetch_class(class_name);
if (!klass) {
klass = Class_singleton(class_name, FILEHANDLE);
}
Class_Override(klass, S_no_op_method, LUCY_FH_Close_OFFSET);
fh = (FileHandle*)Class_Make_Obj(klass);
return FH_do_open(fh, NULL, 0);
}
void
CFWriter_Consolidate_IMP(CompoundFileWriter *self) {
CompoundFileWriterIVARS *const ivars = CFWriter_IVARS(self);
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (Folder_Exists(ivars->folder, cfmeta_file)) {
THROW(ERR, "Merge already performed for %o",
Folder_Get_Path(ivars->folder));
}
else {
S_clean_up_old_temp_files(self, ivars);
S_do_consolidate(self, ivars);
}
}
static void
test_Add_and_Delete(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
Snapshot_Add_Entry(snapshot, foo);
Snapshot_Add_Entry(snapshot, foo); // redundant
Vector *entries = Snapshot_List(snapshot);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1,
"One entry added");
TEST_TRUE(runner, Str_Equals(foo, Vec_Fetch(entries, 0)), "correct entry");
DECREF(entries);
Snapshot_Add_Entry(snapshot, bar);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 2,
"second entry added");
Snapshot_Delete_Entry(snapshot, foo);
TEST_INT_EQ(runner, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry");
DECREF(snapshot);
}
void
TestFH_Run_IMP(TestFileHandle *self, TestBatchRunner *runner) {
TestBatchRunner_Plan(runner, (TestBatch*)self, 2);
FileHandle *fh = S_new_filehandle();
String *foo = SSTR_WRAP_UTF8("foo", 3);
TEST_TRUE(runner, Str_Equals_Utf8(FH_Get_Path(fh), "", 0), "Get_Path");
FH_Set_Path(fh, foo);
TEST_TRUE(runner, Str_Equals(FH_Get_Path(fh), (Obj*)foo), "Set_Path");
DECREF(fh);
}
static void
test_Dump_Load_and_Equals(TestBatchRunner *runner) {
if (!RegexTokenizer_is_available()) {
SKIP(runner, "RegexTokenizer not available");
SKIP(runner, "RegexTokenizer not available");
SKIP(runner, "RegexTokenizer not available");
return;
}
StackString *word_char_pattern = SSTR_WRAP_UTF8("\\w+", 3);
StackString *whitespace_pattern = SSTR_WRAP_UTF8("\\S+", 3);
RegexTokenizer *word_char_tokenizer
= RegexTokenizer_new((String*)word_char_pattern);
RegexTokenizer *whitespace_tokenizer
= RegexTokenizer_new((String*)whitespace_pattern);
Obj *word_char_dump = RegexTokenizer_Dump(word_char_tokenizer);
Obj *whitespace_dump = RegexTokenizer_Dump(whitespace_tokenizer);
RegexTokenizer *word_char_clone
= RegexTokenizer_Load(whitespace_tokenizer, word_char_dump);
RegexTokenizer *whitespace_clone
= RegexTokenizer_Load(whitespace_tokenizer, whitespace_dump);
TEST_FALSE(runner,
RegexTokenizer_Equals(word_char_tokenizer, (Obj*)whitespace_tokenizer),
"Equals() false with different pattern");
TEST_TRUE(runner,
RegexTokenizer_Equals(word_char_tokenizer, (Obj*)word_char_clone),
"Dump => Load round trip");
TEST_TRUE(runner,
RegexTokenizer_Equals(whitespace_tokenizer, (Obj*)whitespace_clone),
"Dump => Load round trip");
DECREF(word_char_tokenizer);
DECREF(word_char_dump);
DECREF(word_char_clone);
DECREF(whitespace_tokenizer);
DECREF(whitespace_dump);
DECREF(whitespace_clone);
}
static void
test_path_handling(TestBatchRunner *runner) {
Snapshot *snapshot = Snapshot_new();
Folder *folder = (Folder*)RAMFolder_new(NULL);
String *snap = (String*)SSTR_WRAP_UTF8("snap", 4);
String *crackle = (String*)SSTR_WRAP_UTF8("crackle", 7);
Snapshot_Write_File(snapshot, folder, snap);
TEST_TRUE(runner, Str_Equals(snap, (Obj*)Snapshot_Get_Path(snapshot)),
"Write_File() sets path as a side effect");
Folder_Rename(folder, snap, crackle);
Snapshot_Read_File(snapshot, folder, crackle);
TEST_TRUE(runner, Str_Equals(crackle, (Obj*)Snapshot_Get_Path(snapshot)),
"Read_File() sets path as a side effect");
Snapshot_Set_Path(snapshot, snap);
TEST_TRUE(runner, Str_Equals(snap, (Obj*)Snapshot_Get_Path(snapshot)),
"Set_Path()");
DECREF(folder);
DECREF(snapshot);
}
bool
Class_add_alias_to_registry(Class *klass, const char *alias_ptr,
size_t alias_len) {
if (Class_registry == NULL) {
Class_init_registry();
}
String *alias = SSTR_WRAP_UTF8(alias_ptr, alias_len);
if (LFReg_fetch(Class_registry, alias)) {
return false;
}
else {
return LFReg_register(Class_registry, alias, (Obj*)klass);
}
}
static void
test_Initialize_and_Check(TestBatchRunner *runner) {
rmdir("_fstest");
String *test_dir = (String*)SSTR_WRAP_UTF8("_fstest", 7);
FSFolder *folder = FSFolder_new(test_dir);
TEST_FALSE(runner, FSFolder_Check(folder),
"Check() returns false when folder dir doesn't exist");
FSFolder_Initialize(folder);
PASS(runner, "Initialize() concludes without incident");
TEST_TRUE(runner, FSFolder_Check(folder),
"Initialize() created dir, and now Check() succeeds");
DECREF(folder);
S_tear_down();
}
static void
test_Window(TestBatchRunner *runner) {
String *test_filename = (String*)SSTR_WRAP_UTF8("_fstest", 7);
FSFileHandle *fh;
FileWindow *window = FileWindow_new();
FileWindowIVARS *const window_ivars = FileWindow_IVARS(window);
uint32_t i;
remove(Str_Get_Ptr8(test_filename));
fh = FSFH_open(test_filename,
FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
for (i = 0; i < 1024; i++) {
FSFH_Write(fh, "foo ", 4);
}
if (!FSFH_Close(fh)) { RETHROW(INCREF(Err_get_error())); }
// Reopen for reading.
DECREF(fh);
fh = FSFH_open(test_filename, FH_READ_ONLY);
if (!fh) { RETHROW(INCREF(Err_get_error())); }
Err_set_error(NULL);
TEST_FALSE(runner, FSFH_Window(fh, window, -1, 4),
"Window() with a negative offset returns false");
TEST_TRUE(runner, Err_get_error() != NULL,
"Window() with a negative offset sets error");
Err_set_error(NULL);
TEST_FALSE(runner, FSFH_Window(fh, window, 4000, 1000),
"Window() past EOF returns false");
TEST_TRUE(runner, Err_get_error() != NULL,
"Window() past EOF sets error");
TEST_TRUE(runner, FSFH_Window(fh, window, 1021, 2),
"Window() returns true");
TEST_TRUE(runner,
strncmp(window_ivars->buf - window_ivars->offset + 1021, "oo", 2) == 0,
"Window()");
TEST_TRUE(runner, FSFH_Release_Window(fh, window),
"Release_Window() returns true");
TEST_TRUE(runner, window_ivars->buf == NULL, "Release_Window() resets buf");
TEST_TRUE(runner, window_ivars->offset == 0, "Release_Window() resets offset");
TEST_TRUE(runner, window_ivars->len == 0, "Release_Window() resets len");
DECREF(window);
DECREF(fh);
remove(Str_Get_Ptr8(test_filename));
}
Folder*
FSFolder_Local_Find_Folder_IMP(FSFolder *self, String *name) {
FSFolderIVARS *const ivars = FSFolder_IVARS(self);
Folder *subfolder = NULL;
if (!name || !Str_Get_Size(name)) {
// No entity can be identified by NULL or empty string.
return NULL;
}
else if (!S_is_local_entry(name)) {
return NULL;
}
else if (Str_Starts_With_Utf8(name, ".", 1)) {
// Don't allow access outside of the main dir.
return NULL;
}
else if (NULL != (subfolder = (Folder*)Hash_Fetch(ivars->entries, (Obj*)name))) {
if (Folder_Is_A(subfolder, FOLDER)) {
return subfolder;
}
else {
return NULL;
}
}
String *fullpath = S_fullpath(self, name);
if (S_dir_ok(fullpath)) {
subfolder = (Folder*)FSFolder_new(fullpath);
if (!subfolder) {
DECREF(fullpath);
THROW(ERR, "Failed to open FSFolder at '%o'", fullpath);
}
// Try to open a CompoundFileReader. On failure, just use the
// existing folder.
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (Folder_Local_Exists(subfolder, cfmeta_file)) {
CompoundFileReader *cf_reader = CFReader_open(subfolder);
if (cf_reader) {
DECREF(subfolder);
subfolder = (Folder*)cf_reader;
}
}
Hash_Store(ivars->entries, (Obj*)name, (Obj*)subfolder);
}
DECREF(fullpath);
return subfolder;
}
static void
S_zap_dead_merge(FilePurger *self, Hash *candidates) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
IndexManager *manager = ivars->manager;
Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
Lock_Clear_Stale(merge_lock);
if (!Lock_Is_Locked(merge_lock)) {
Hash *merge_data = IxManager_Read_Merge_Data(manager);
Obj *cutoff = merge_data
? Hash_Fetch_Utf8(merge_data, "cutoff", 6)
: NULL;
if (cutoff) {
String *cutoff_seg = Seg_num_to_name(Json_obj_to_i64(cutoff));
if (Folder_Exists(ivars->folder, cutoff_seg)) {
String *merge_json = SSTR_WRAP_UTF8("merge.json", 10);
DirHandle *dh = Folder_Open_Dir(ivars->folder, cutoff_seg);
if (!dh) {
THROW(ERR, "Can't open segment dir '%o'", cutoff_seg);
}
Hash_Store(candidates, cutoff_seg, (Obj*)CFISH_TRUE);
Hash_Store(candidates, merge_json, (Obj*)CFISH_TRUE);
while (DH_Next(dh)) {
// TODO: recursively delete subdirs within seg dir.
String *entry = DH_Get_Entry(dh);
String *filepath = Str_newf("%o/%o", cutoff_seg, entry);
Hash_Store(candidates, filepath, (Obj*)CFISH_TRUE);
DECREF(filepath);
DECREF(entry);
}
DECREF(dh);
}
DECREF(cutoff_seg);
}
DECREF(merge_data);
}
DECREF(merge_lock);
return;
}
