static void
S_read_fsfolder(RAMFolder *self)
{
u32_t i, max;
/* Open an FSFolder for reading. */
FSFolder *source_folder = FSFolder_new(self->path);
VArray *files = FSFolder_List(source_folder);
/* Copy every file in the FSFolder into RAM. */
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *filepath = (CharBuf*)VA_Fetch(files, i);
InStream *source_stream
= FSFolder_Open_In(source_folder, filepath);
OutStream *outstream = RAMFolder_Open_Out(self, filepath);
if (!source_stream) { THROW("Can't open %o", filepath); }
if (!outstream) { THROW("Can't open %o", filepath); }
OutStream_Absorb(outstream, source_stream);
OutStream_Close(outstream);
InStream_Close(source_stream);
DECREF(outstream);
DECREF(source_stream);
}
DECREF(files);
FSFolder_Close(source_folder);
DECREF(source_folder);
}
static Folder*
S_init_folder(Obj *index, bool_t create)
{
Folder *folder = NULL;
// Validate or acquire a Folder.
if (Obj_Is_A(index, FOLDER)) {
folder = (Folder*)INCREF(index);
}
else if (Obj_Is_A(index, CHARBUF)) {
folder = (Folder*)FSFolder_new((CharBuf*)index);
}
else {
THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
}
// Validate or create the index directory.
if (create) {
Folder_Initialize(folder);
}
else {
if (!Folder_Check(folder)) {
THROW(ERR, "Folder '%o' failed check", Folder_Get_Path(folder));
}
}
return folder;
}
static Folder*
S_init_folder(Obj *index, bool create) {
Folder *folder = NULL;
// Validate or acquire a Folder.
if (Obj_is_a(index, FOLDER)) {
folder = (Folder*)INCREF(index);
}
else if (Obj_is_a(index, STRING)) {
folder = (Folder*)FSFolder_new((String*)index);
}
else {
THROW(ERR, "Invalid type for 'index': %o", Obj_get_class_name(index));
}
// Validate or create the index directory.
if (create) {
Folder_Initialize(folder);
}
else {
if (!Folder_Check(folder)) {
THROW(ERR, "Folder '%o' failed check", Folder_Get_Path(folder));
}
}
return folder;
}
void
Indexer_add_index(Indexer *self, Obj *index)
{
Folder *other_folder = NULL;
IndexReader *reader = NULL;
if (Obj_Is_A(index, FOLDER)) {
other_folder = (Folder*)INCREF(index);
}
else if (Obj_Is_A(index, CHARBUF)) {
other_folder = (Folder*)FSFolder_new((CharBuf*)index);
}
else {
THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
}
reader = IxReader_open((Obj*)other_folder, NULL, NULL);
if (reader == NULL) {
THROW(ERR, "Index doesn't seem to contain any data");
}
else {
Schema *schema = self->schema;
Schema *other_schema = IxReader_Get_Schema(reader);
VArray *other_fields = Schema_All_Fields(other_schema);
VArray *seg_readers = IxReader_Seg_Readers(reader);
uint32_t i, max;
// Validate schema compatibility and add fields.
Schema_Eat(schema, other_schema);
// Add fields to Segment.
for (i = 0, max = VA_Get_Size(other_fields); i < max; i++) {
CharBuf *other_field = (CharBuf*)VA_Fetch(other_fields, i);
Seg_Add_Field(self->segment, other_field);
}
DECREF(other_fields);
// Add all segments.
for (i = 0, max = VA_Get_Size(seg_readers); i < max; i++) {
SegReader *seg_reader = (SegReader*)VA_Fetch(seg_readers, i);
DeletionsReader *del_reader = (DeletionsReader*)SegReader_Fetch(
seg_reader, VTable_Get_Name(DELETIONSREADER));
Matcher *deletions = del_reader
? DelReader_Iterator(del_reader)
: NULL;
I32Array *doc_map = DelWriter_Generate_Doc_Map(self->del_writer,
deletions, SegReader_Doc_Max(seg_reader),
(int32_t)Seg_Get_Count(self->segment)
);
SegWriter_Add_Segment(self->seg_writer, seg_reader, doc_map);
DECREF(deletions);
DECREF(doc_map);
}
DECREF(seg_readers);
}
DECREF(reader);
DECREF(other_folder);
}
void
Indexer_Add_Index_IMP(Indexer *self, Obj *index) {
IndexerIVARS *const ivars = Indexer_IVARS(self);
Folder *other_folder = NULL;
IndexReader *reader = NULL;
if (Obj_is_a(index, FOLDER)) {
other_folder = (Folder*)INCREF(index);
}
else if (Obj_is_a(index, STRING)) {
other_folder = (Folder*)FSFolder_new((String*)index);
}
else {
THROW(ERR, "Invalid type for 'index': %o", Obj_get_class_name(index));
}
reader = IxReader_open((Obj*)other_folder, NULL, NULL);
if (reader == NULL) {
THROW(ERR, "Index doesn't seem to contain any data");
}
else {
Schema *schema = ivars->schema;
Schema *other_schema = IxReader_Get_Schema(reader);
Vector *other_fields = Schema_All_Fields(other_schema);
Vector *seg_readers = IxReader_Seg_Readers(reader);
// Validate schema compatibility and add fields.
Schema_Eat(schema, other_schema);
// Add fields to Segment.
for (size_t i = 0, max = Vec_Get_Size(other_fields); i < max; i++) {
String *other_field = (String*)Vec_Fetch(other_fields, i);
Seg_Add_Field(ivars->segment, other_field);
}
DECREF(other_fields);
// Add all segments.
for (size_t i = 0, max = Vec_Get_Size(seg_readers); i < max; i++) {
SegReader *seg_reader = (SegReader*)Vec_Fetch(seg_readers, i);
DeletionsReader *del_reader
= (DeletionsReader*)SegReader_Fetch(
seg_reader, Class_Get_Name(DELETIONSREADER));
Matcher *deletions = del_reader
? DelReader_Iterator(del_reader)
: NULL;
I32Array *doc_map = DelWriter_Generate_Doc_Map(
ivars->del_writer, deletions,
SegReader_Doc_Max(seg_reader),
(int32_t)Seg_Get_Count(ivars->segment));
SegWriter_Add_Segment(ivars->seg_writer, seg_reader, doc_map);
DECREF(deletions);
DECREF(doc_map);
}
DECREF(seg_readers);
}
DECREF(reader);
DECREF(other_folder);
}
static void
test_all(TestBatch *batch) {
CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
CharBuf *foo_boffo = (CharBuf*)ZCB_WRAP_STR("foo/boffo", 9);
CharBuf *test_dir = (CharBuf*)ZCB_WRAP_STR("_fsdir_test", 11);
FSFolder *folder = FSFolder_new(test_dir);
bool_t saw_foo = false;
bool_t saw_boffo = false;
bool_t foo_was_dir = false;
bool_t boffo_was_dir = false;
int count = 0;
// Clean up after previous failed runs.
FSFolder_Delete(folder, foo_boffo);
FSFolder_Delete(folder, foo);
FSFolder_Delete(folder, boffo);
rmdir("_fsdir_test");
FSFolder_Initialize(folder);
FSFolder_MkDir(folder, foo);
OutStream *outstream = FSFolder_Open_Out(folder, boffo);
DECREF(outstream);
outstream = FSFolder_Open_Out(folder, foo_boffo);
DECREF(outstream);
FSDirHandle *dh = FSDH_open(test_dir);
CharBuf *entry = FSDH_Get_Entry(dh);
while (FSDH_Next(dh)) {
count++;
if (CB_Equals(entry, (Obj*)foo)) {
saw_foo = true;
foo_was_dir = FSDH_Entry_Is_Dir(dh);
}
else if (CB_Equals(entry, (Obj*)boffo)) {
saw_boffo = true;
boffo_was_dir = FSDH_Entry_Is_Dir(dh);
}
}
TEST_INT_EQ(batch, 2, count, "correct number of entries");
TEST_TRUE(batch, saw_foo, "Directory was iterated over");
TEST_TRUE(batch, foo_was_dir,
"Dir correctly identified by Entry_Is_Dir");
TEST_TRUE(batch, saw_boffo, "File was iterated over");
TEST_FALSE(batch, boffo_was_dir,
"File correctly identified by Entry_Is_Dir");
DECREF(dh);
FSFolder_Delete(folder, foo_boffo);
FSFolder_Delete(folder, foo);
FSFolder_Delete(folder, boffo);
DECREF(folder);
rmdir("_fsdir_test");
}
static Folder*
S_set_up() {
rmdir("_fstest");
String *test_dir = (String*)SSTR_WRAP_UTF8("_fstest", 7);
FSFolder *folder = FSFolder_new(test_dir);
FSFolder_Initialize(folder);
if (!FSFolder_Check(folder)) {
RETHROW(INCREF(Err_get_error()));
}
return (Folder*)folder;
}
static Folder*
S_derive_folder(Obj *index) {
Folder *folder = NULL;
if (Obj_Is_A(index, FOLDER)) {
folder = (Folder*)INCREF(index);
}
else if (Obj_Is_A(index, CHARBUF)) {
folder = (Folder*)FSFolder_new((CharBuf*)index);
}
else {
THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
}
return folder;
}
static void
test_Initialize_and_Check(TestBatchRunner *runner) {
rmdir("_fstest");
String *test_dir = (String*)SSTR_WRAP_UTF8("_fstest", 7);
FSFolder *folder = FSFolder_new(test_dir);
TEST_FALSE(runner, FSFolder_Check(folder),
"Check() returns false when folder dir doesn't exist");
FSFolder_Initialize(folder);
PASS(runner, "Initialize() concludes without incident");
TEST_TRUE(runner, FSFolder_Check(folder),
"Initialize() created dir, and now Check() succeeds");
DECREF(folder);
S_tear_down();
}
Folder*
FSFolder_Local_Find_Folder_IMP(FSFolder *self, String *name) {
FSFolderIVARS *const ivars = FSFolder_IVARS(self);
Folder *subfolder = NULL;
if (!name || !Str_Get_Size(name)) {
// No entity can be identified by NULL or empty string.
return NULL;
}
else if (!S_is_local_entry(name)) {
return NULL;
}
else if (Str_Starts_With_Utf8(name, ".", 1)) {
// Don't allow access outside of the main dir.
return NULL;
}
else if (NULL != (subfolder = (Folder*)Hash_Fetch(ivars->entries, (Obj*)name))) {
if (Folder_Is_A(subfolder, FOLDER)) {
return subfolder;
}
else {
return NULL;
}
}
String *fullpath = S_fullpath(self, name);
if (S_dir_ok(fullpath)) {
subfolder = (Folder*)FSFolder_new(fullpath);
if (!subfolder) {
DECREF(fullpath);
THROW(ERR, "Failed to open FSFolder at '%o'", fullpath);
}
// Try to open a CompoundFileReader. On failure, just use the
// existing folder.
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (Folder_Local_Exists(subfolder, cfmeta_file)) {
CompoundFileReader *cf_reader = CFReader_open(subfolder);
if (cf_reader) {
DECREF(subfolder);
subfolder = (Folder*)cf_reader;
}
}
Hash_Store(ivars->entries, (Obj*)name, (Obj*)subfolder);
}
DECREF(fullpath);
return subfolder;
}
void
test_disallow_updir(TestBatchRunner *runner) {
FSFolder *outer_folder = (FSFolder*)S_set_up();
String *foo = (String*)SSTR_WRAP_UTF8("foo", 3);
String *bar = (String*)SSTR_WRAP_UTF8("bar", 3);
FSFolder_MkDir(outer_folder, foo);
FSFolder_MkDir(outer_folder, bar);
String *inner_path = (String*)SSTR_WRAP_UTF8("_fstest/foo", 11);
FSFolder *foo_folder = FSFolder_new(inner_path);
String *up_bar = (String*)SSTR_WRAP_UTF8("../bar", 6);
TEST_FALSE(runner, FSFolder_Exists(foo_folder, up_bar),
"up-dirs are inaccessible.");
DECREF(foo_folder);
FSFolder_Delete(outer_folder, foo);
FSFolder_Delete(outer_folder, bar);
DECREF(outer_folder);
S_tear_down();
}
