Segment*
IxManager_make_new_segment(IndexManager *self, Snapshot *snapshot)
{
VArray *files = Snapshot_List(snapshot);
u32_t i, max;
i32_t highest_seg_num = 0;
CharBuf *seg_name = CB_new(20);
Segment *segment;
/* Find highest seg num. */
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(files, i);
if (CB_Starts_With_Str(file, "seg_", 4)) {
i32_t seg_num = IxFileNames_extract_gen(file);
if (seg_num > highest_seg_num) {
highest_seg_num = seg_num;
}
}
}
/* Create segment with num one greater than current max. */
S_cat_seg_name(seg_name, highest_seg_num + 1);
segment = Seg_new(seg_name, self->folder);
DECREF(seg_name);
DECREF(files);
return segment;
}
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
VArray *files = Folder_List(self->folder);
u32_t i, max;
i32_t max_gen = 0;
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(files, i);
if ( CB_Starts_With_Str(file, "snapshot_", 9)
&& CB_Ends_With_Str(file, ".json", 5)
) {
i32_t gen = IxFileNames_extract_gen(file);
if (gen > max_gen) {
max_gen = gen;
}
}
}
DECREF(files);
{
i32_t new_gen = max_gen + 1;
CharBuf *base_36 = StrHelp_to_base36(new_gen);
CharBuf *snapfile = CB_newf("snapshot_%o.json", base_36);
DECREF(base_36);
return snapfile;
}
}
CharBuf*
IxManager_make_snapshot_filename(IndexManager *self)
{
Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
DirHandle *dh = Folder_Open_Dir(folder, NULL);
CharBuf *entry;
uint64_t max_gen = 0;
if (!dh) { RETHROW(INCREF(Err_get_error())); }
entry = DH_Get_Entry(dh);
while (DH_Next(dh)) {
if ( CB_Starts_With_Str(entry, "snapshot_", 9)
&& CB_Ends_With_Str(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > max_gen) { max_gen = gen; }
}
}
DECREF(dh);
{
uint64_t new_gen = max_gen + 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(new_gen, &base36);
return CB_newf("snapshot_%s.json", &base36);
}
}
bool_t
Seg_valid_seg_name(const CharBuf *name) {
if (CB_Starts_With_Str(name, "seg_", 4)) {
ZombieCharBuf *scratch = ZCB_WRAP(name);
ZCB_Nip(scratch, 4);
uint32_t code_point;
while (0 != (code_point = ZCB_Nip_One(scratch))) {
if (!isalnum(code_point)) { return false; }
}
if (ZCB_Get_Size(scratch) == 0) { return true; } // Success!
}
return false;
}
static void
test_To_String(TestBatch *batch) {
Float32 *f32 = Float32_new(1.33f);
Float64 *f64 = Float64_new(1.33);
Integer32 *i32 = Int32_new(I32_MAX);
Integer64 *i64 = Int64_new(I64_MAX);
CharBuf *f32_string = Float32_To_String(f32);
CharBuf *f64_string = Float64_To_String(f64);
CharBuf *i32_string = Int32_To_String(i32);
CharBuf *i64_string = Int64_To_String(i64);
CharBuf *true_string = Bool_To_String(CFISH_TRUE);
CharBuf *false_string = Bool_To_String(CFISH_FALSE);
TEST_TRUE(batch, CB_Starts_With_Str(f32_string, "1.3", 3),
"Float32_To_String");
TEST_TRUE(batch, CB_Starts_With_Str(f64_string, "1.3", 3),
"Float64_To_String");
TEST_TRUE(batch, CB_Equals_Str(i32_string, "2147483647", 10),
"Int32_To_String");
TEST_TRUE(batch, CB_Equals_Str(i64_string, "9223372036854775807", 19),
"Int64_To_String");
TEST_TRUE(batch, CB_Equals_Str(true_string, "true", 4),
"Bool_To_String [true]");
TEST_TRUE(batch, CB_Equals_Str(false_string, "false", 5),
"Bool_To_String [false]");
DECREF(false_string);
DECREF(true_string);
DECREF(i64_string);
DECREF(i32_string);
DECREF(f64_string);
DECREF(f32_string);
DECREF(i64);
DECREF(i32);
DECREF(f64);
DECREF(f32);
}
Lock*
IxManager_make_snapshot_read_lock(IndexManager *self, const CharBuf *filename)
{
ZombieCharBuf *lock_name = ZCB_WRAP(filename);
LockFactory *lock_factory = S_obtain_lock_factory(self);
if ( !CB_Starts_With_Str(filename, "snapshot_", 9)
|| !CB_Ends_With_Str(filename, ".json", 5)
) {
THROW(ERR, "Not a snapshot filename: %o", filename);
}
// Truncate ".json" from end of snapshot file name.
ZCB_Chop(lock_name, sizeof(".json") - 1);
return LockFact_Make_Shared_Lock(lock_factory, (CharBuf*)lock_name, 1000, 100);
}
static VArray*
S_clean_segment_contents(VArray *orig)
{
// Since Snapshot format 2, no DataReader has depended on individual files
// within segment directories being listed. Filter these files because
// they cause a problem with FilePurger.
VArray *cleaned = VA_new(VA_Get_Size(orig));
for (uint32_t i = 0, max = VA_Get_Size(orig); i < max; i++) {
CharBuf *name = (CharBuf*)VA_Fetch(orig, i);
if (!Seg_valid_seg_name(name)) {
if (CB_Starts_With_Str(name, "seg_", 4)) {
continue; // Skip this file.
}
}
VA_Push(cleaned, INCREF(name));
}
return cleaned;
}
static CharBuf*
S_find_schema_file(Snapshot *snapshot)
{
VArray *files = Snapshot_List(snapshot);
uint32_t i, max;
CharBuf *retval = NULL;
for (i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(files, i);
if ( CB_Starts_With_Str(file, "schema_", 7)
&& CB_Ends_With_Str(file, ".json", 5)
) {
retval = file;
break;
}
}
DECREF(files);
return retval;
}
void
S_try_open_elements(void *context) {
struct try_open_elements_context *args
= (struct try_open_elements_context*)context;
PolyReader *self = args->self;
PolyReaderIVARS *const ivars = PolyReader_IVARS(self);
VArray *files = Snapshot_List(ivars->snapshot);
Folder *folder = PolyReader_Get_Folder(self);
uint32_t num_segs = 0;
uint64_t latest_schema_gen = 0;
CharBuf *schema_file = NULL;
// Find schema file, count segments.
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
if (Seg_valid_seg_name(entry)) {
num_segs++;
}
else if (CB_Starts_With_Str(entry, "schema_", 7)
&& CB_Ends_With_Str(entry, ".json", 5)
) {
uint64_t gen = IxFileNames_extract_gen(entry);
if (gen > latest_schema_gen) {
latest_schema_gen = gen;
if (!schema_file) { schema_file = CB_Clone(entry); }
else { CB_Mimic(schema_file, (Obj*)entry); }
}
}
}
// Read Schema.
if (!schema_file) {
DECREF(files);
THROW(ERR, "Can't find a schema file.");
}
else {
Hash *dump = (Hash*)Json_slurp_json(folder, schema_file);
if (dump) { // read file successfully
DECREF(ivars->schema);
ivars->schema = (Schema*)CERTIFY(
VTable_Load_Obj(SCHEMA, (Obj*)dump), SCHEMA);
DECREF(dump);
DECREF(schema_file);
schema_file = NULL;
}
else {
CharBuf *mess = MAKE_MESS("Failed to parse %o", schema_file);
DECREF(schema_file);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
VArray *segments = VA_new(num_segs);
for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
// Create a Segment for each segmeta.
if (Seg_valid_seg_name(entry)) {
int64_t seg_num = IxFileNames_extract_gen(entry);
Segment *segment = Seg_new(seg_num);
// Bail if reading the file fails (probably because it's been
// deleted and a new snapshot file has been written so we need to
// retry).
if (Seg_Read_File(segment, folder)) {
VA_Push(segments, (Obj*)segment);
}
else {
CharBuf *mess = MAKE_MESS("Failed to read %o", entry);
DECREF(segment);
DECREF(segments);
DECREF(files);
Err_throw_mess(ERR, mess);
}
}
}
// Sort the segments by age.
VA_Sort(segments, NULL, NULL);
// Open individual SegReaders.
struct try_open_segreader_context seg_context;
seg_context.schema = PolyReader_Get_Schema(self);
seg_context.folder = folder;
seg_context.snapshot = PolyReader_Get_Snapshot(self);
seg_context.segments = segments;
seg_context.result = NULL;
args->seg_readers = VA_new(num_segs);
Err *error = NULL;
for (uint32_t seg_tick = 0; seg_tick < num_segs; seg_tick++) {
seg_context.seg_tick = seg_tick;
error = Err_trap(S_try_open_segreader, &seg_context);
if (error) {
break;
}
VA_Push(args->seg_readers, (Obj*)seg_context.result);
seg_context.result = NULL;
}
DECREF(segments);
DECREF(files);
if (error) {
DECREF(args->seg_readers);
args->seg_readers = NULL;
RETHROW(error);
}
}
static void
S_discover_unused(FilePurger *self, VArray **purgables_ptr,
VArray **snapshots_ptr)
{
Folder *folder = self->folder;
DirHandle *dh = Folder_Open_Dir(folder, NULL);
if (!dh) { RETHROW(INCREF(Err_get_error())); }
VArray *spared = VA_new(1);
VArray *snapshots = VA_new(1);
CharBuf *snapfile = NULL;
// Start off with the list of files in the current snapshot.
if (self->snapshot) {
VArray *entries = Snapshot_List(self->snapshot);
VArray *referenced = S_find_all_referenced(folder, entries);
VA_Push_VArray(spared, referenced);
DECREF(entries);
DECREF(referenced);
snapfile = Snapshot_Get_Path(self->snapshot);
if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
}
CharBuf *entry = DH_Get_Entry(dh);
Hash *candidates = Hash_new(64);
while (DH_Next(dh)) {
if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; }
else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; }
else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
else {
Snapshot *snapshot
= Snapshot_Read_File(Snapshot_new(), folder, entry);
Lock *lock
= IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
VArray *snap_list = Snapshot_List(snapshot);
VArray *referenced = S_find_all_referenced(folder, snap_list);
// DON'T obtain the lock -- only see whether another
// entity holds a lock on the snapshot file.
if (lock) {
Lock_Clear_Stale(lock);
}
if (lock && Lock_Is_Locked(lock)) {
// The snapshot file is locked, which means someone's using
// that version of the index -- protect all of its entries.
uint32_t new_size = VA_Get_Size(spared)
+ VA_Get_Size(referenced) + 1;
VA_Grow(spared, new_size);
VA_Push(spared, (Obj*)CB_Clone(entry));
VA_Push_VArray(spared, referenced);
}
else {
// No one's using this snapshot, so all of its entries are
// candidates for deletion.
for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
}
VA_Push(snapshots, INCREF(snapshot));
}
DECREF(referenced);
DECREF(snap_list);
DECREF(snapshot);
DECREF(lock);
}
}
DECREF(dh);
// Clean up after a dead segment consolidation.
S_zap_dead_merge(self, candidates);
// Eliminate any current files from the list of files to be purged.
for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
DECREF(Hash_Delete(candidates, (Obj*)filename));
}
// Pass back purgables and Snapshots.
*purgables_ptr = Hash_Keys(candidates);
*snapshots_ptr = snapshots;
DECREF(candidates);
DECREF(spared);
}
