QueryParser*
QParser_init(QueryParser *self, Schema *schema, Analyzer *analyzer,
String *default_boolop, Vector *fields) {
QueryParserIVARS *const ivars = QParser_IVARS(self);
// Init.
ivars->heed_colons = false;
ivars->lexer = QueryLexer_new();
// Assign.
ivars->schema = (Schema*)INCREF(schema);
ivars->analyzer = (Analyzer*)INCREF(analyzer);
ivars->default_boolop = default_boolop
? Str_Clone(default_boolop)
: Str_new_from_trusted_utf8("OR", 2);
if (fields) {
ivars->fields = Vec_Clone(fields);
for (uint32_t i = 0, max = Vec_Get_Size(fields); i < max; i++) {
CERTIFY(Vec_Fetch(fields, i), STRING);
}
Vec_Sort(ivars->fields);
}
else {
Vector *all_fields = Schema_All_Fields(schema);
uint32_t num_fields = Vec_Get_Size(all_fields);
ivars->fields = Vec_new(num_fields);
for (uint32_t i = 0; i < num_fields; i++) {
String *field = (String*)Vec_Fetch(all_fields, i);
FieldType *type = Schema_Fetch_Type(schema, field);
if (type && FType_Indexed(type)) {
Vec_Push(ivars->fields, INCREF(field));
}
}
DECREF(all_fields);
}
Vec_Sort(ivars->fields);
// Derive default "occur" from default boolean operator.
if (Str_Equals_Utf8(ivars->default_boolop, "OR", 2)) {
ivars->default_occur = SHOULD;
}
else if (Str_Equals_Utf8(ivars->default_boolop, "AND", 3)) {
ivars->default_occur = MUST;
}
else {
THROW(ERR, "Invalid value for default_boolop: %o", ivars->default_boolop);
}
return self;
}
HeatMap*
HeatMap_init(HeatMap *self, Vector *spans, uint32_t window) {
HeatMapIVARS *const ivars = HeatMap_IVARS(self);
Vector *spans_copy = Vec_Clone(spans);
Vector *spans_plus_boosts;
ivars->spans = NULL;
ivars->window = window;
Vec_Sort(spans_copy);
spans_plus_boosts = HeatMap_Generate_Proximity_Boosts(self, spans_copy);
Vec_Push_All(spans_plus_boosts, spans_copy);
Vec_Sort(spans_plus_boosts);
ivars->spans = HeatMap_Flatten_Spans(self, spans_plus_boosts);
DECREF(spans_plus_boosts);
DECREF(spans_copy);
return self;
}
void
Snapshot_Write_File_IMP(Snapshot *self, Folder *folder, String *path) {
SnapshotIVARS *const ivars = Snapshot_IVARS(self);
Hash *all_data = Hash_new(0);
Vector *list = Snapshot_List(self);
// Update path.
DECREF(ivars->path);
if (path != NULL && Str_Get_Size(path) != 0) {
ivars->path = Str_Clone(path);
}
else {
String *latest = IxFileNames_latest_snapshot(folder);
uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
char base36[StrHelp_MAX_BASE36_BYTES];
StrHelp_to_base36(gen, &base36);
ivars->path = Str_newf("snapshot_%s.json", &base36);
DECREF(latest);
}
// Don't overwrite.
if (Folder_Exists(folder, ivars->path)) {
THROW(ERR, "Snapshot file '%o' already exists", ivars->path);
}
// Sort, then store file names.
Vec_Sort(list);
Hash_Store_Utf8(all_data, "entries", 7, (Obj*)list);
// Create a JSON-izable data structure.
Hash_Store_Utf8(all_data, "format", 6,
(Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_format));
Hash_Store_Utf8(all_data, "subformat", 9,
(Obj*)Str_newf("%i32", (int32_t)Snapshot_current_file_subformat));
// Write out JSON-ized data to the new file.
Json_spew_json((Obj*)all_data, folder, ivars->path);
DECREF(all_data);
}
static void
S_do_consolidate(CompoundFileWriter *self, CompoundFileWriterIVARS *ivars) {
UNUSED_VAR(self);
Folder *folder = ivars->folder;
Hash *metadata = Hash_new(0);
Hash *sub_files = Hash_new(0);
Vector *files = Folder_List(folder, NULL);
Vector *merged = Vec_new(Vec_Get_Size(files));
String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6);
OutStream *outstream = Folder_Open_Out(folder, (String*)cf_file);
bool rename_success;
if (!outstream) { RETHROW(INCREF(Err_get_error())); }
// Start metadata.
Hash_Store_Utf8(metadata, "files", 5, INCREF(sub_files));
Hash_Store_Utf8(metadata, "format", 6,
(Obj*)Str_newf("%i32", CFWriter_current_file_format));
Vec_Sort(files);
for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) {
String *infilename = (String*)Vec_Fetch(files, i);
if (!Str_Ends_With_Utf8(infilename, ".json", 5)) {
InStream *instream = Folder_Open_In(folder, infilename);
Hash *file_data = Hash_new(2);
int64_t offset, len;
if (!instream) { RETHROW(INCREF(Err_get_error())); }
// Absorb the file.
offset = OutStream_Tell(outstream);
OutStream_Absorb(outstream, instream);
len = OutStream_Tell(outstream) - offset;
// Record offset and length.
Hash_Store_Utf8(file_data, "offset", 6,
(Obj*)Str_newf("%i64", offset));
Hash_Store_Utf8(file_data, "length", 6,
(Obj*)Str_newf("%i64", len));
Hash_Store(sub_files, infilename, (Obj*)file_data);
Vec_Push(merged, INCREF(infilename));
// Add filler NULL bytes so that every sub-file begins on a file
// position multiple of 8.
OutStream_Align(outstream, 8);
InStream_Close(instream);
DECREF(instream);
}
}
// Write metadata to cfmeta file.
String *cfmeta_temp = (String*)SSTR_WRAP_UTF8("cfmeta.json.temp", 16);
String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11);
Json_spew_json((Obj*)metadata, (Folder*)ivars->folder, cfmeta_temp);
rename_success = Folder_Rename(ivars->folder, cfmeta_temp, cfmeta_file);
if (!rename_success) { RETHROW(INCREF(Err_get_error())); }
// Clean up.
OutStream_Close(outstream);
DECREF(outstream);
DECREF(files);
DECREF(metadata);
/*
HashIterator *iter = HashIter_new(sub_files);
while (HashIter_Next(iter)) {
String *merged_file = HashIter_Get_Key(iter);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(sub_files);
Err_throw_mess(ERR, mess);
}
}
DECREF(iter);
*/
DECREF(sub_files);
for (uint32_t i = 0, max = Vec_Get_Size(merged); i < max; i++) {
String *merged_file = (String*)Vec_Fetch(merged, i);
if (!Folder_Delete(folder, merged_file)) {
String *mess = MAKE_MESS("Can't delete '%o'", merged_file);
DECREF(merged);
Err_throw_mess(ERR, mess);
}
}
DECREF(merged);
}
void
FilePurger_Purge_IMP(FilePurger *self) {
FilePurgerIVARS *const ivars = FilePurger_IVARS(self);
Lock *deletion_lock = IxManager_Make_Deletion_Lock(ivars->manager);
// Obtain deletion lock, purge files, release deletion lock.
Lock_Clear_Stale(deletion_lock);
if (Lock_Obtain(deletion_lock)) {
Folder *folder = ivars->folder;
Hash *failures = Hash_new(0);
Vector *purgables;
Vector *snapshots;
S_discover_unused(self, &purgables, &snapshots);
// Attempt to delete entries -- if failure, no big deal, just try
// again later. Proceed in reverse lexical order so that directories
// get deleted after they've been emptied.
Vec_Sort(purgables);
for (uint32_t i = Vec_Get_Size(purgables); i--;) {
String *entry = (String*)Vec_Fetch(purgables, i);
if (Hash_Fetch(ivars->disallowed, entry)) { continue; }
if (!Folder_Delete(folder, entry)) {
if (Folder_Exists(folder, entry)) {
Hash_Store(failures, entry, (Obj*)CFISH_TRUE);
}
}
}
for (uint32_t i = 0, max = Vec_Get_Size(snapshots); i < max; i++) {
Snapshot *snapshot = (Snapshot*)Vec_Fetch(snapshots, i);
bool snapshot_has_failures = false;
if (Hash_Get_Size(failures)) {
// Only delete snapshot files if all of their entries were
// successfully deleted.
Vector *entries = Snapshot_List(snapshot);
for (uint32_t j = Vec_Get_Size(entries); j--;) {
String *entry = (String*)Vec_Fetch(entries, j);
if (Hash_Fetch(failures, entry)) {
snapshot_has_failures = true;
break;
}
}
DECREF(entries);
}
if (!snapshot_has_failures) {
String *snapfile = Snapshot_Get_Path(snapshot);
Folder_Delete(folder, snapfile);
}
}
DECREF(failures);
DECREF(purgables);
DECREF(snapshots);
Lock_Release(deletion_lock);
}
else {
WARN("Can't obtain deletion lock, skipping deletion of "
"obsolete files");
}
DECREF(deletion_lock);
}
