bool_t
RAMFolder_local_delete(RAMFolder *self, const CharBuf *name)
{
Obj *entry = Hash_Fetch(self->entries, (Obj*)name);
if (entry) {
if (Obj_Is_A(entry, RAMFILE)) {
;
}
else if (Obj_Is_A(entry, FOLDER)) {
RAMFolder *inner_folder;
if (Obj_Is_A(entry, COMPOUNDFILEREADER)) {
inner_folder = (RAMFolder*)CERTIFY(
CFReader_Get_Real_Folder((CompoundFileReader*)entry),
RAMFOLDER);
}
else {
inner_folder = (RAMFolder*)CERTIFY(entry, RAMFOLDER);
}
if (Hash_Get_Size(inner_folder->entries)) {
// Can't delete non-empty dir.
return false;
}
}
else {
return false;
}
DECREF(Hash_Delete(self->entries, (Obj*)name));
return true;
}
else {
return false;
}
}
bool
CFReader_Local_Delete_IMP(CompoundFileReader *self, String *name) {
CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self);
Hash *record = (Hash*)Hash_Delete(ivars->records, name);
DECREF(record);
if (record == NULL) {
return Folder_Local_Delete(ivars->real_folder, name);
}
else {
// Once the number of virtual files falls to 0, remove the compound
// files.
if (Hash_Get_Size(ivars->records) == 0) {
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
if (!Folder_Delete(ivars->real_folder, cf_file)) {
return false;
}
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
if (!Folder_Delete(ivars->real_folder, cfmeta_file)) {
return false;
}
}
return true;
}
}
bool_t
CFReader_local_delete(CompoundFileReader *self, const CharBuf *name) {
Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name);
DECREF(record);
if (record == NULL) {
return Folder_Local_Delete(self->real_folder, name);
}
else {
// Once the number of virtual files falls to 0, remove the compound
// files.
if (Hash_Get_Size(self->records) == 0) {
CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
if (!Folder_Delete(self->real_folder, cf_file)) {
return false;
}
CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
if (!Folder_Delete(self->real_folder, cfmeta_file)) {
return false;
}
}
return true;
}
}
Obj*
S_dump_hash(Hash *hash) {
Hash *dump = Hash_new(Hash_Get_Size(hash));
HashIterator *iter = HashIter_new(hash);
while (HashIter_Next(iter)) {
String *key = HashIter_Get_Key(iter);
Obj *value = HashIter_Get_Value(iter);
Hash_Store(dump, key, Freezer_dump(value));
}
DECREF(iter);
return (Obj*)dump;
}
void
Freezer_serialize_hash(Hash *hash, OutStream *outstream) {
uint32_t hash_size = Hash_Get_Size(hash);
OutStream_Write_C32(outstream, hash_size);
HashIterator *iter = HashIter_new(hash);
while (HashIter_Next(iter)) {
String *key = HashIter_Get_Key(iter);
Obj *val = HashIter_Get_Value(iter);
Freezer_serialize_string(key, outstream);
FREEZE(val, outstream);
}
DECREF(iter);
}
Hash*
Schema_Dump_IMP(Schema *self) {
SchemaIVARS *const ivars = Schema_IVARS(self);
Hash *dump = Hash_new(0);
Hash *type_dumps = Hash_new(Hash_Get_Size(ivars->types));
// Record class name, store dumps of unique Analyzers.
Hash_Store_Utf8(dump, "_class", 6,
(Obj*)Str_Clone(Schema_get_class_name(self)));
Hash_Store_Utf8(dump, "analyzers", 9,
Freezer_dump((Obj*)ivars->uniq_analyzers));
// Dump FieldTypes.
Hash_Store_Utf8(dump, "fields", 6, (Obj*)type_dumps);
HashIterator *iter = HashIter_new(ivars->types);
while (HashIter_Next(iter)) {
String *field = HashIter_Get_Key(iter);
FieldType *type = (FieldType*)HashIter_Get_Value(iter);
Class *type_class = FType_get_class(type);
// Dump known types to simplified format.
if (type_class == FULLTEXTTYPE) {
FullTextType *fttype = (FullTextType*)type;
Hash *type_dump = FullTextType_Dump_For_Schema(fttype);
Analyzer *analyzer = FullTextType_Get_Analyzer(fttype);
uint32_t tick
= S_find_in_array(ivars->uniq_analyzers, (Obj*)analyzer);
// Store the tick which references a unique analyzer.
Hash_Store_Utf8(type_dump, "analyzer", 8,
(Obj*)Str_newf("%u32", tick));
Hash_Store(type_dumps, field, (Obj*)type_dump);
}
else if (type_class == STRINGTYPE || type_class == BLOBTYPE) {
Hash *type_dump = FType_Dump_For_Schema(type);
Hash_Store(type_dumps, field, (Obj*)type_dump);
}
// Unknown FieldType type, so punt.
else {
Hash_Store(type_dumps, field, FType_Dump(type));
}
}
DECREF(iter);
return dump;
}
static Obj*
S_load_from_hash(Hash *dump) {
String *class_name = (String*)Hash_Fetch_Utf8(dump, "_class", 6);
// Assume that the presence of the "_class" key paired with a valid class
// name indicates the output of a dump() rather than an ordinary Hash.
if (class_name && Str_is_a(class_name, STRING)) {
Class *klass = Class_fetch_class(class_name);
if (!klass) {
String *parent_class_name = Class_find_parent_class(class_name);
if (parent_class_name) {
Class *parent = Class_singleton(parent_class_name, NULL);
klass = Class_singleton(class_name, parent);
DECREF(parent_class_name);
}
else {
// TODO: Fix load() so that it works with ordinary hash keys
// named "_class".
THROW(ERR, "Can't find class '%o'", class_name);
}
}
// Dispatch to an alternate Load() method.
if (klass) {
return S_load_via_load_method(klass, (Obj*)dump);
}
}
// It's an ordinary Hash.
Hash *loaded = Hash_new(Hash_Get_Size(dump));
HashIterator *iter = HashIter_new(dump);
while (HashIter_Next(iter)) {
String *key = HashIter_Get_Key(iter);
Obj *value = HashIter_Get_Value(iter);
Hash_Store(loaded, key, Freezer_load(value));
}
DECREF(iter);
return (Obj*)loaded;
}
Hash*
LexWriter_metadata(LexiconWriter *self)
{
Hash *const metadata = DataWriter_metadata((DataWriter*)self);
Hash *const counts = (Hash*)INCREF(self->counts);
Hash *const ix_counts = (Hash*)INCREF(self->ix_counts);
/* Placeholders. */
if (Hash_Get_Size(counts) == 0) {
Hash_Store_Str(counts, "none", 4, (Obj*)CB_newf("%i32", (i32_t)0) );
Hash_Store_Str(ix_counts, "none", 4,
(Obj*)CB_newf("%i32", (i32_t)0) );
}
Hash_Store_Str(metadata, "counts", 6, (Obj*)counts);
Hash_Store_Str(metadata, "index_counts", 12, (Obj*)ix_counts);
return metadata;
}
static void
test_offsets(TestBatchRunner *runner) {
Folder *folder = S_folder_with_contents();
CompoundFileWriter *cf_writer = CFWriter_new(folder);
Hash *cf_metadata;
Hash *files;
CFWriter_Consolidate(cf_writer);
cf_metadata = (Hash*)CERTIFY(
Json_slurp_json(folder, cfmeta_file), HASH);
files = (Hash*)CERTIFY(
Hash_Fetch_Utf8(cf_metadata, "files", 5), HASH);
bool offsets_ok = true;
TEST_TRUE(runner, Hash_Get_Size(files) > 0, "Multiple files");
HashIterator *iter = HashIter_new(files);
while (HashIter_Next(iter)) {
String *file = HashIter_Get_Key(iter);
Hash *stats = (Hash*)CERTIFY(HashIter_Get_Value(iter), HASH);
Obj *offset = CERTIFY(Hash_Fetch_Utf8(stats, "offset", 6), OBJ);
int64_t offs = Obj_To_I64(offset);
if (offs % 8 != 0) {
offsets_ok = false;
FAIL(runner, "Offset %" PRId64 " for %s not a multiple of 8",
offset, Str_Get_Ptr8(file));
break;
}
}
DECREF(iter);
if (offsets_ok) {
PASS(runner, "All offsets are multiples of 8");
}
DECREF(cf_metadata);
DECREF(cf_writer);
DECREF(folder);
}
static void
test_Store_and_Fetch(TestBatch *batch) {
Hash *hash = Hash_new(100);
Hash *dupe = Hash_new(100);
const uint32_t starting_cap = Hash_Get_Capacity(hash);
VArray *expected = VA_new(100);
VArray *got = VA_new(100);
ZombieCharBuf *twenty = ZCB_WRAP_STR("20", 2);
ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2);
ZombieCharBuf *foo = ZCB_WRAP_STR("foo", 3);
for (int32_t i = 0; i < 100; i++) {
CharBuf *cb = CB_newf("%i32", i);
Hash_Store(hash, (Obj*)cb, (Obj*)cb);
Hash_Store(dupe, (Obj*)cb, INCREF(cb));
VA_Push(expected, INCREF(cb));
}
TEST_TRUE(batch, Hash_Equals(hash, (Obj*)dupe), "Equals");
TEST_INT_EQ(batch, Hash_Get_Capacity(hash), starting_cap,
"Initial capacity sufficient (no rebuilds)");
for (int32_t i = 0; i < 100; i++) {
Obj *key = VA_Fetch(expected, i);
Obj *elem = Hash_Fetch(hash, key);
VA_Push(got, (Obj*)INCREF(elem));
}
TEST_TRUE(batch, VA_Equals(got, (Obj*)expected),
"basic Store and Fetch");
TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
"size incremented properly by Hash_Store");
TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)foo) == NULL,
"Fetch against non-existent key returns NULL");
Hash_Store(hash, (Obj*)forty, INCREF(foo));
TEST_TRUE(batch, ZCB_Equals(foo, Hash_Fetch(hash, (Obj*)forty)),
"Hash_Store replaces existing value");
TEST_FALSE(batch, Hash_Equals(hash, (Obj*)dupe),
"replacement value spoils equals");
TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
"size unaffected after value replaced");
TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == (Obj*)foo,
"Delete returns value");
DECREF(foo);
TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
"size decremented by successful Delete");
TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == NULL,
"Delete returns NULL when key not found");
TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
"size not decremented by unsuccessful Delete");
DECREF(Hash_Delete(dupe, (Obj*)forty));
TEST_TRUE(batch, VA_Equals(got, (Obj*)expected), "Equals after Delete");
Hash_Clear(hash);
TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)twenty) == NULL, "Clear");
TEST_TRUE(batch, Hash_Get_Size(hash) == 0, "size is 0 after Clear");
DECREF(hash);
DECREF(dupe);
DECREF(got);
DECREF(expected);
}
uint32_t
Schema_Num_Fields_IMP(Schema *self) {
SchemaIVARS *const ivars = Schema_IVARS(self);
return Hash_Get_Size(ivars->types);
}
uint32_t
Snapshot_Num_Entries_IMP(Snapshot *self) {
SnapshotIVARS *const ivars = Snapshot_IVARS(self);
return Hash_Get_Size(ivars->entries);
}
u32_t
Snapshot_num_entries(Snapshot *self) { return Hash_Get_Size(self->entries); }
static bool_t
S_to_json(Obj *dump, CharBuf *json, int32_t depth) {
// Guard against infinite recursion in self-referencing data structures.
if (depth > MAX_DEPTH) {
CharBuf *mess = MAKE_MESS("Exceeded max depth of %i32", MAX_DEPTH);
Err_set_error(Err_new(mess));
return false;
}
if (!dump) {
CB_Cat_Trusted_Str(json, "null", 4);
}
else if (dump == (Obj*)CFISH_TRUE) {
CB_Cat_Trusted_Str(json, "true", 4);
}
else if (dump == (Obj*)CFISH_FALSE) {
CB_Cat_Trusted_Str(json, "false", 5);
}
else if (Obj_Is_A(dump, CHARBUF)) {
S_append_json_string(dump, json);
}
else if (Obj_Is_A(dump, INTNUM)) {
CB_catf(json, "%i64", Obj_To_I64(dump));
}
else if (Obj_Is_A(dump, FLOATNUM)) {
CB_catf(json, "%f64", Obj_To_F64(dump));
}
else if (Obj_Is_A(dump, VARRAY)) {
VArray *array = (VArray*)dump;
size_t size = VA_Get_Size(array);
if (size == 0) {
// Put empty array on single line.
CB_Cat_Trusted_Str(json, "[]", 2);
return true;
}
else if (size == 1) {
Obj *elem = VA_Fetch(array, 0);
if (!(Obj_Is_A(elem, HASH) || Obj_Is_A(elem, VARRAY))) {
// Put array containing single scalar element on one line.
CB_Cat_Trusted_Str(json, "[", 1);
if (!S_to_json(elem, json, depth + 1)) {
return false;
}
CB_Cat_Trusted_Str(json, "]", 1);
return true;
}
}
// Fall back to spreading elements across multiple lines.
CB_Cat_Trusted_Str(json, "[", 1);
for (size_t i = 0; i < size; i++) {
CB_Cat_Trusted_Str(json, "\n", 1);
S_cat_whitespace(json, depth + 1);
if (!S_to_json(VA_Fetch(array, i), json, depth + 1)) {
return false;
}
if (i + 1 < size) {
CB_Cat_Trusted_Str(json, ",", 1);
}
}
CB_Cat_Trusted_Str(json, "\n", 1);
S_cat_whitespace(json, depth);
CB_Cat_Trusted_Str(json, "]", 1);
}
else if (Obj_Is_A(dump, HASH)) {
Hash *hash = (Hash*)dump;
size_t size = Hash_Get_Size(hash);
// Put empty hash on single line.
if (size == 0) {
CB_Cat_Trusted_Str(json, "{}", 2);
return true;
}
// Validate that all keys are strings, then sort.
VArray *keys = Hash_Keys(hash);
for (size_t i = 0; i < size; i++) {
Obj *key = VA_Fetch(keys, i);
if (!key || !Obj_Is_A(key, CHARBUF)) {
DECREF(keys);
CharBuf *key_class = key ? Obj_Get_Class_Name(key) : NULL;
CharBuf *mess = MAKE_MESS("Illegal key type: %o", key_class);
Err_set_error(Err_new(mess));
return false;
}
}
VA_Sort(keys, NULL, NULL);
// Spread pairs across multiple lines.
CB_Cat_Trusted_Str(json, "{", 1);
for (size_t i = 0; i < size; i++) {
Obj *key = VA_Fetch(keys, i);
CB_Cat_Trusted_Str(json, "\n", 1);
S_cat_whitespace(json, depth + 1);
S_append_json_string(key, json);
CB_Cat_Trusted_Str(json, ": ", 2);
if (!S_to_json(Hash_Fetch(hash, key), json, depth + 1)) {
DECREF(keys);
return false;
}
if (i + 1 < size) {
CB_Cat_Trusted_Str(json, ",", 1);
}
}
CB_Cat_Trusted_Str(json, "\n", 1);
S_cat_whitespace(json, depth);
CB_Cat_Trusted_Str(json, "}", 1);
DECREF(keys);
}
return true;
}
void
FilePurger_purge(FilePurger *self)
{
Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager);
// Obtain deletion lock, purge files, release deletion lock.
Lock_Clear_Stale(deletion_lock);
if (Lock_Obtain(deletion_lock)) {
Folder *folder = self->folder;
Hash *failures = Hash_new(0);
VArray *purgables;
VArray *snapshots;
S_discover_unused(self, &purgables, &snapshots);
// Attempt to delete entries -- if failure, no big deal, just try
// again later. Proceed in reverse lexical order so that directories
// get deleted after they've been emptied.
VA_Sort(purgables, NULL, NULL);
for (uint32_t i = VA_Get_Size(purgables); i--; ) {
CharBuf *entry = (CharBuf*)VA_fetch(purgables, i);
if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; }
if (!Folder_Delete(folder, entry)) {
if (Folder_Exists(folder, entry)) {
Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY));
}
}
}
for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) {
Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i);
bool_t snapshot_has_failures = false;
if (Hash_Get_Size(failures)) {
// Only delete snapshot files if all of their entries were
// successfully deleted.
VArray *entries = Snapshot_List(snapshot);
for (uint32_t j = VA_Get_Size(entries); j--; ) {
CharBuf *entry = (CharBuf*)VA_Fetch(entries, j);
if (Hash_Fetch(failures, (Obj*)entry)) {
snapshot_has_failures = true;
break;
}
}
DECREF(entries);
}
if (!snapshot_has_failures) {
CharBuf *snapfile = Snapshot_Get_Path(snapshot);
Folder_Delete(folder, snapfile);
}
}
DECREF(failures);
DECREF(purgables);
DECREF(snapshots);
Lock_Release(deletion_lock);
}
else {
WARN("Can't obtain deletion lock, skipping deletion of "
"obsolete files");
}
DECREF(deletion_lock);
}
uint32_t
Doc_Get_Size_IMP(Doc *self) {
Hash *hash = (Hash*)Doc_IVARS(self)->fields;
return Hash_Get_Size(hash);
}