InStream* CFReader_local_open_in(CompoundFileReader *self, const CharBuf *name) { Hash *entry = (Hash*)Hash_Fetch(self->records, (Obj*)name); if (!entry) { InStream *instream = Folder_Local_Open_In(self->real_folder, name); if (!instream) { ERR_ADD_FRAME(Err_get_error()); } return instream; } else { Obj *len = Hash_Fetch_Str(entry, "length", 6); Obj *offset = Hash_Fetch_Str(entry, "offset", 6); if (!len || !offset) { Err_set_error(Err_new(CB_newf("Malformed entry for '%o' in '%o'", name, Folder_Get_Path(self->real_folder)))); return NULL; } else if (CB_Get_Size(self->path)) { CharBuf *fullpath = CB_newf("%o/%o", self->path, name); InStream *instream = InStream_Reopen(self->instream, fullpath, Obj_To_I64(offset), Obj_To_I64(len)); DECREF(fullpath); return instream; } else { return InStream_Reopen(self->instream, name, Obj_To_I64(offset), Obj_To_I64(len)); } } }
CharBuf* Json_to_json(Obj *dump) { // Validate object type, only allowing hashes and arrays per JSON spec. if (!dump || !(Obj_Is_A(dump, HASH) || Obj_Is_A(dump, VARRAY))) { if (!tolerant) { CharBuf *class_name = dump ? Obj_Get_Class_Name(dump) : NULL; CharBuf *mess = MAKE_MESS("Illegal top-level object type: %o", class_name); Err_set_error(Err_new(mess)); return NULL; } } // Encode. CharBuf *json = CB_new(31); if (!S_to_json(dump, json, 0)) { DECREF(json); ERR_ADD_FRAME(Err_get_error()); json = NULL; } else { // Append newline. CB_Cat_Trusted_Str(json, "\n", 1); } return json; }
InStream* CFReader_Local_Open_In_IMP(CompoundFileReader *self, String *name) { CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self); Hash *entry = (Hash*)Hash_Fetch(ivars->records, name); if (!entry) { InStream *instream = Folder_Local_Open_In(ivars->real_folder, name); if (!instream) { ERR_ADD_FRAME(Err_get_error()); } return instream; } else { Obj *len = Hash_Fetch_Utf8(entry, "length", 6); Obj *offset = Hash_Fetch_Utf8(entry, "offset", 6); if (!len || !offset) { Err_set_error(Err_new(Str_newf("Malformed entry for '%o' in '%o'", name, Folder_Get_Path(ivars->real_folder)))); return NULL; } else if (Str_Get_Size(ivars->path)) { String *fullpath = Str_newf("%o/%o", ivars->path, name); InStream *instream = InStream_Reopen(ivars->instream, fullpath, Obj_To_I64(offset), Obj_To_I64(len)); DECREF(fullpath); return instream; } else { return InStream_Reopen(ivars->instream, name, Obj_To_I64(offset), Obj_To_I64(len)); } } }
/* This method exists as a hook for CompoundFileReader to override; it is * necessary because calling CFReader_Local_Open_FileHandle() won't find * virtual files. No other class should need to override it. */ InStream* Folder_local_open_in(Folder *self, const CharBuf *name) { FileHandle *fh = Folder_Local_Open_FileHandle(self, name, FH_READ_ONLY); InStream *instream = NULL; if (fh) { instream = InStream_open((Obj*)fh); DECREF(fh); if (!instream) { ERR_ADD_FRAME(Err_get_error()); } } else { ERR_ADD_FRAME(Err_get_error()); } return instream; }
DirHandle* RAMFolder_local_open_dir(RAMFolder *self) { RAMDirHandle *dh = RAMDH_new(self); if (!dh) { ERR_ADD_FRAME(Err_get_error()); } return (DirHandle*)dh; }
DirHandle* FSFolder_Local_Open_Dir_IMP(FSFolder *self) { FSFolderIVARS *const ivars = FSFolder_IVARS(self); DirHandle *dh = (DirHandle*)FSDH_open(ivars->path); if (!dh) { ERR_ADD_FRAME(Err_get_error()); } return dh; }
bool FSFolder_Local_MkDir_IMP(FSFolder *self, String *name) { String *dir = S_fullpath(self, name); bool result = S_create_dir(dir); if (!result) { ERR_ADD_FRAME(Err_get_error()); } DECREF(dir); return result; }
OutStream* Folder_open_out(Folder *self, const CharBuf *path) { const uint32_t flags = FH_WRITE_ONLY | FH_CREATE | FH_EXCLUSIVE; FileHandle *fh = Folder_Open_FileHandle(self, path, flags); OutStream *outstream = NULL; if (fh) { outstream = OutStream_open((Obj*)fh); DECREF(fh); if (!outstream) { ERR_ADD_FRAME(Err_get_error()); } } else { ERR_ADD_FRAME(Err_get_error()); } return outstream; }
Obj* Json_slurp_json(Folder *folder, const CharBuf *path) { InStream *instream = Folder_Open_In(folder, path); if (!instream) { ERR_ADD_FRAME(Err_get_error()); return NULL; } size_t len = (size_t)InStream_Length(instream); char *buf = InStream_Buf(instream, len); Obj *dump = S_parse_json(buf, len); InStream_Close(instream); DECREF(instream); if (!dump) { ERR_ADD_FRAME(Err_get_error()); } return dump; }
Obj* Json_from_json(CharBuf *json) { Obj *dump = S_parse_json((char*)CB_Get_Ptr8(json), CB_Get_Size(json)); if (!dump) { ERR_ADD_FRAME(Err_get_error()); } return dump; }
InStream* InStream_do_open(InStream *self, Obj *file) { InStreamIVARS *const ivars = InStream_IVARS(self); // Init. ivars->buf = NULL; ivars->limit = NULL; ivars->offset = 0; ivars->window = FileWindow_new(); // Obtain a FileHandle. if (Obj_Is_A(file, FILEHANDLE)) { ivars->file_handle = (FileHandle*)INCREF(file); } else if (Obj_Is_A(file, RAMFILE)) { ivars->file_handle = (FileHandle*)RAMFH_open(NULL, FH_READ_ONLY, (RAMFile*)file); } else if (Obj_Is_A(file, CHARBUF)) { ivars->file_handle = (FileHandle*)FSFH_open((CharBuf*)file, FH_READ_ONLY); } else { Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'", Obj_Get_Class_Name(file)))); DECREF(self); return NULL; } if (!ivars->file_handle) { ERR_ADD_FRAME(Err_get_error()); DECREF(self); return NULL; } // Get length and filename from the FileHandle. ivars->filename = CB_Clone(FH_Get_Path(ivars->file_handle)); ivars->len = FH_Length(ivars->file_handle); if (ivars->len == -1) { ERR_ADD_FRAME(Err_get_error()); DECREF(self); return NULL; } return self; }
FileHandle* FSFolder_Local_Open_FileHandle_IMP(FSFolder *self, String *name, uint32_t flags) { String *fullpath = S_fullpath(self, name); FSFileHandle *fh = FSFH_open(fullpath, flags); if (!fh) { ERR_ADD_FRAME(Err_get_error()); } DECREF(fullpath); return (FileHandle*)fh; }
bool_t Json_spew_json(Obj *dump, Folder *folder, const CharBuf *path) { CharBuf *json = Json_to_json(dump); if (!json) { ERR_ADD_FRAME(Err_get_error()); return false; } OutStream *outstream = Folder_Open_Out(folder, path); if (!outstream) { ERR_ADD_FRAME(Err_get_error()); DECREF(json); return false; } size_t size = CB_Get_Size(json); OutStream_Write_Bytes(outstream, CB_Get_Ptr8(json), size); OutStream_Close(outstream); DECREF(outstream); DECREF(json); return true; }
bool_t RAMFolder_hard_link(RAMFolder *self, const CharBuf *from, const CharBuf *to) { Folder *from_folder = RAMFolder_Enclosing_Folder(self, from); Folder *to_folder = RAMFolder_Enclosing_Folder(self, to); ZombieCharBuf *from_name = IxFileNames_local_part(from, ZCB_BLANK()); ZombieCharBuf *to_name = IxFileNames_local_part(to, ZCB_BLANK()); bool_t result = S_rename_or_hard_link(self, from, to, from_folder, to_folder, from_name, to_name, OP_HARD_LINK); if (!result) { ERR_ADD_FRAME(Err_get_error()); } return result; }
bool_t CFReader_local_mkdir(CompoundFileReader *self, const CharBuf *name) { if (Hash_Fetch(self->records, (Obj*)name)) { Err_set_error(Err_new(CB_newf("Can't MkDir: '%o' exists", name))); return false; } else { bool_t result = Folder_Local_MkDir(self->real_folder, name); if (!result) { ERR_ADD_FRAME(Err_get_error()); } return result; } }
bool CFReader_Local_MkDir_IMP(CompoundFileReader *self, String *name) { CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self); if (Hash_Fetch(ivars->records, name)) { Err_set_error(Err_new(Str_newf("Can't MkDir: '%o' exists", name))); return false; } else { bool result = Folder_Local_MkDir(ivars->real_folder, name); if (!result) { ERR_ADD_FRAME(Err_get_error()); } return result; } }
bool_t Lock_obtain(Lock *self) { int32_t time_left = self->interval == 0 ? 0 : self->timeout; bool_t locked = Lock_Request(self); while (!locked) { time_left -= self->interval; if (time_left <= 0) { break; } Sleep_millisleep(self->interval); locked = Lock_Request(self); } if (!locked) { ERR_ADD_FRAME(Err_get_error()); } return locked; }
VArray* Folder_list(Folder *self, const CharBuf *path) { Folder *local_folder = Folder_Find_Folder(self, path); VArray *list = NULL; DirHandle *dh = Folder_Local_Open_Dir(local_folder); if (dh) { CharBuf *entry = DH_Get_Entry(dh); list = VA_new(32); while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); } DECREF(dh); } else { ERR_ADD_FRAME(Err_get_error()); } return list; }
bool Lock_Obtain_IMP(Lock *self) { LockIVARS *const ivars = Lock_IVARS(self); int32_t time_left = ivars->interval == 0 ? 0 : ivars->timeout; bool locked = Lock_Request(self); while (!locked) { time_left -= ivars->interval; if (time_left <= 0) { break; } Sleep_millisleep(ivars->interval); locked = Lock_Request(self); } if (!locked) { ERR_ADD_FRAME(Err_get_error()); } return locked; }
InStream* Folder_open_in(Folder *self, const CharBuf *path) { Folder *enclosing_folder = Folder_Enclosing_Folder(self, path); InStream *instream = NULL; if (enclosing_folder) { ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK()); instream = Folder_Local_Open_In(enclosing_folder, (CharBuf*)name); if (!instream) { ERR_ADD_FRAME(Err_get_error()); } } else { Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path))); } return instream; }
FileHandle* RAMFolder_local_open_filehandle(RAMFolder *self, const CharBuf *name, uint32_t flags) { RAMFileHandle *fh; CharBuf *fullpath = S_fullpath(self, name); RAMFile *file = (RAMFile*)Hash_Fetch(self->entries, (Obj*)name); bool_t can_create = (flags & (FH_WRITE_ONLY | FH_CREATE)) == (FH_WRITE_ONLY | FH_CREATE) ? true : false; // Make sure the filepath isn't a directory, and that it either exists // or we have permission to create it. if (file) { if (!RAMFile_Is_A(file, RAMFILE)) { Err_set_error(Err_new(CB_newf("Not a file: '%o'", fullpath))); DECREF(fullpath); return NULL; } } else if (!can_create) { Err_set_error(Err_new(CB_newf("File not found: '%o'", fullpath))); DECREF(fullpath); return NULL; } // Open the file and store it if it was just created. fh = RAMFH_open(fullpath, flags, file); if (fh) { if (!file) { file = RAMFH_Get_File(fh); Hash_Store(self->entries, (Obj*)name, INCREF(file)); } } else { Err *error = Err_get_error(); ERR_ADD_FRAME(error); } DECREF(fullpath); return (FileHandle*)fh; }
FileHandle* CFReader_local_open_filehandle(CompoundFileReader *self, const CharBuf *name, uint32_t flags) { Hash *entry = (Hash*)Hash_Fetch(self->records, (Obj*)name); FileHandle *fh = NULL; if (entry) { Err_set_error(Err_new(CB_newf("Can't open FileHandle for virtual file %o in '%o'", name, self->path))); } else { fh = Folder_Local_Open_FileHandle(self->real_folder, name, flags); if (!fh) { ERR_ADD_FRAME(Err_get_error()); } } return fh; }
FileHandle* Folder_open_filehandle(Folder *self, const CharBuf *path, uint32_t flags) { Folder *enclosing_folder = Folder_Enclosing_Folder(self, path); FileHandle *fh = NULL; if (enclosing_folder) { ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK()); fh = Folder_Local_Open_FileHandle(enclosing_folder, (CharBuf*)name, flags); if (!fh) { ERR_ADD_FRAME(Err_get_error()); } } else { Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path))); } return fh; }
FileHandle* CFReader_Local_Open_FileHandle_IMP(CompoundFileReader *self, String *name, uint32_t flags) { CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self); Hash *entry = (Hash*)Hash_Fetch(ivars->records, name); FileHandle *fh = NULL; if (entry) { Err_set_error(Err_new(Str_newf("Can't open FileHandle for virtual file %o in '%o'", name, ivars->path))); } else { fh = Folder_Local_Open_FileHandle(ivars->real_folder, name, flags); if (!fh) { ERR_ADD_FRAME(Err_get_error()); } } return fh; }
bool Folder_mkdir(Folder *self, const CharBuf *path) { Folder *enclosing_folder = Folder_Enclosing_Folder(self, path); bool result = false; if (!CB_Get_Size(path)) { Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path))); } else if (!enclosing_folder) { Err_set_error(Err_new(CB_newf("Can't recursively create dir %o", path))); } else { ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK()); result = Folder_Local_MkDir(enclosing_folder, (CharBuf*)name); if (!result) { ERR_ADD_FRAME(Err_get_error()); } } return result; }
DirHandle* Folder_open_dir(Folder *self, const CharBuf *path) { DirHandle *dh = NULL; Folder *folder; if (path) { folder = Folder_Find_Folder(self, path); } else { ZombieCharBuf *empty = ZCB_BLANK(); folder = Folder_Find_Folder(self, (CharBuf*)empty); } if (!folder) { Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path))); } else { dh = Folder_Local_Open_Dir(folder); if (!dh) { ERR_ADD_FRAME(Err_get_error()); } } return dh; }
OutStream* OutStream_do_open(OutStream *self, Obj *file) { OutStreamIVARS *const ivars = OutStream_IVARS(self); // Init. ivars->buf = (char*)MALLOCATE(IO_STREAM_BUF_SIZE); ivars->buf_start = 0; ivars->buf_pos = 0; // Obtain a FileHandle. if (Obj_is_a(file, FILEHANDLE)) { ivars->file_handle = (FileHandle*)INCREF(file); } else if (Obj_is_a(file, RAMFILE)) { ivars->file_handle = (FileHandle*)RAMFH_open(NULL, FH_WRITE_ONLY, (RAMFile*)file); } else if (Obj_is_a(file, STRING)) { ivars->file_handle = (FileHandle*)FSFH_open((String*)file, FH_WRITE_ONLY | FH_CREATE | FH_EXCLUSIVE); } else { Err_set_error(Err_new(Str_newf("Invalid type for param 'file': '%o'", Obj_get_class_name(file)))); DECREF(self); return NULL; } if (!ivars->file_handle) { ERR_ADD_FRAME(Err_get_error()); DECREF(self); return NULL; } // Derive filepath from FileHandle. ivars->path = Str_Clone(FH_Get_Path(ivars->file_handle)); return self; }
static Obj* S_do_parse_json(void *json_parser, char *json, size_t len) { lucy_JsonParserState state; state.result = NULL; state.errors = false; char *text = json; char *const end = text + len; while (text < end) { int token_type = -1; Obj *value = NULL; char *const save = text; switch (*text) { case ' ': case '\n': case '\r': case '\t': // Skip insignificant whitespace, which the JSON RFC defines // as only four ASCII characters. text++; continue; case '[': token_type = LUCY_JSON_TOKENTYPE_LEFT_SQUARE_BRACKET; text++; break; case ']': token_type = LUCY_JSON_TOKENTYPE_RIGHT_SQUARE_BRACKET; text++; break; case '{': token_type = LUCY_JSON_TOKENTYPE_LEFT_CURLY_BRACKET; text++; break; case '}': token_type = LUCY_JSON_TOKENTYPE_RIGHT_CURLY_BRACKET; text++; break; case ':': token_type = LUCY_JSON_TOKENTYPE_COLON; text++; break; case ',': token_type = LUCY_JSON_TOKENTYPE_COMMA; text++; break; case '"': value = (Obj*)S_parse_string(&text, end); if (value) { token_type = LUCY_JSON_TOKENTYPE_STRING; } else { // Clear out parser and return. LucyParseJson(json_parser, 0, NULL, &state); ERR_ADD_FRAME(Err_get_error()); return NULL; } break; case 'n': if (SI_check_keyword(text, end, "null", 4)) { token_type = LUCY_JSON_TOKENTYPE_NULL; text += 4; } break; case 't': if (SI_check_keyword(text, end, "true", 4)) { token_type = LUCY_JSON_TOKENTYPE_TRUE; value = (Obj*)CFISH_TRUE; text += 4; } break; case 'f': if (SI_check_keyword(text, end, "false", 5)) { token_type = LUCY_JSON_TOKENTYPE_FALSE; value = (Obj*)CFISH_FALSE; text += 5; } break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': case '-': { // Note no '+', as JSON spec doesn't allow it. value = (Obj*)S_parse_number(&text, end); if (value) { token_type = LUCY_JSON_TOKENTYPE_NUMBER; } else { // Clear out parser and return. LucyParseJson(json_parser, 0, NULL, &state); ERR_ADD_FRAME(Err_get_error()); return NULL; } } break; } LucyParseJson(json_parser, token_type, value, &state); if (state.errors) { SET_ERROR(CB_newf("JSON syntax error"), save, end); return NULL; } } // Finish up. LucyParseJson(json_parser, 0, NULL, &state); if (state.errors) { SET_ERROR(CB_newf("JSON syntax error"), json, end); return NULL; } return state.result; }
bool LFLock_Request_IMP(LockFileLock *self) { LockFileLockIVARS *const ivars = LFLock_IVARS(self); bool success = false; if (Folder_Exists(ivars->folder, ivars->lock_path)) { Err_set_error((Err*)LockErr_new(Str_newf("Can't obtain lock: '%o' exists", ivars->lock_path))); return false; } // Create the "locks" subdirectory if necessary. String *lock_dir_name = (String*)SSTR_WRAP_UTF8("locks", 5); if (!Folder_Exists(ivars->folder, lock_dir_name)) { if (!Folder_MkDir(ivars->folder, lock_dir_name)) { Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR); LockErr *err = LockErr_new(Str_newf("Can't create 'locks' directory: %o", Err_Get_Mess(mkdir_err))); // Maybe our attempt failed because another process succeeded. if (Folder_Find_Folder(ivars->folder, lock_dir_name)) { DECREF(err); } else { // Nope, everything failed, so bail out. Err_set_error((Err*)err); return false; } } } // Prepare to write pid, lock name, and host to the lock file as JSON. Hash *file_data = Hash_new(3); Hash_Store_Utf8(file_data, "pid", 3, (Obj*)Str_newf("%i32", (int32_t)PID_getpid())); Hash_Store_Utf8(file_data, "host", 4, INCREF(ivars->host)); Hash_Store_Utf8(file_data, "name", 4, INCREF(ivars->name)); String *json = Json_to_json((Obj*)file_data); DECREF(file_data); // Write to a temporary file, then use the creation of a hard link to // ensure atomic but non-destructive creation of the lockfile with its // complete contents. OutStream *outstream = Folder_Open_Out(ivars->folder, ivars->link_path); if (!outstream) { ERR_ADD_FRAME(Err_get_error()); DECREF(json); return false; } struct lockfile_context context; context.outstream = outstream; context.json = json; Err *json_error = Err_trap(S_write_lockfile_json, &context); bool wrote_json = !json_error; DECREF(outstream); DECREF(json); if (wrote_json) { success = Folder_Hard_Link(ivars->folder, ivars->link_path, ivars->lock_path); if (!success) { Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR); Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o", ivars->lock_path, Err_Get_Mess(hard_link_err)))); } } else { Err_set_error((Err*)LockErr_new(Str_newf("Failed to obtain lock at '%o': %o", ivars->lock_path, Err_Get_Mess(json_error)))); DECREF(json_error); } // Verify that our temporary file got zapped. bool deletion_failed = !Folder_Delete(ivars->folder, ivars->link_path); if (deletion_failed) { String *mess = MAKE_MESS("Failed to delete '%o'", ivars->link_path); Err_throw_mess(ERR, mess); } return success; }
CompoundFileReader* CFReader_do_open(CompoundFileReader *self, Folder *folder) { CompoundFileReaderIVARS *const ivars = CFReader_IVARS(self); String *cfmeta_file = (String*)SSTR_WRAP_UTF8("cfmeta.json", 11); Hash *metadata = (Hash*)Json_slurp_json((Folder*)folder, cfmeta_file); Err *error = NULL; Folder_init((Folder*)self, Folder_Get_Path(folder)); // Parse metadata file. if (!metadata || !Hash_Is_A(metadata, HASH)) { error = Err_new(Str_newf("Can't read '%o' in '%o'", cfmeta_file, Folder_Get_Path(folder))); } else { Obj *format = Hash_Fetch_Utf8(metadata, "format", 6); ivars->format = format ? (int32_t)Obj_To_I64(format) : 0; ivars->records = (Hash*)INCREF(Hash_Fetch_Utf8(metadata, "files", 5)); if (ivars->format < 1) { error = Err_new(Str_newf("Corrupt %o file: Missing or invalid 'format'", cfmeta_file)); } else if (ivars->format > CFWriter_current_file_format) { error = Err_new(Str_newf("Unsupported compound file format: %i32 " "(current = %i32", ivars->format, CFWriter_current_file_format)); } else if (!ivars->records) { error = Err_new(Str_newf("Corrupt %o file: missing 'files' key", cfmeta_file)); } } DECREF(metadata); if (error) { Err_set_error(error); DECREF(self); return NULL; } // Open an instream which we'll clone over and over. String *cf_file = (String*)SSTR_WRAP_UTF8("cf.dat", 6); ivars->instream = Folder_Open_In(folder, cf_file); if (!ivars->instream) { ERR_ADD_FRAME(Err_get_error()); DECREF(self); return NULL; } // Assign. ivars->real_folder = (Folder*)INCREF(folder); // Strip directory name from filepaths for old format. if (ivars->format == 1) { Vector *files = Hash_Keys(ivars->records); String *folder_name = IxFileNames_local_part(Folder_Get_Path(folder)); size_t folder_name_len = Str_Length(folder_name); for (uint32_t i = 0, max = Vec_Get_Size(files); i < max; i++) { String *orig = (String*)Vec_Fetch(files, i); if (Str_Starts_With(orig, folder_name)) { Obj *record = Hash_Delete(ivars->records, orig); size_t offset = folder_name_len + sizeof(CHY_DIR_SEP) - 1; size_t len = Str_Length(orig) - offset; String *filename = Str_SubString(orig, offset, len); Hash_Store(ivars->records, filename, (Obj*)record); DECREF(filename); } } DECREF(folder_name); DECREF(files); } return self; }